Search in sources :

Example 1 with ExceptionUtils.getStackTrace

use of org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace in project bitsquare by bitsquare.

the class SeedNodeMain method doExecute.

@Override
protected void doExecute(OptionSet options) {
    final BitsquareEnvironment environment = new BitsquareEnvironment(options);
    SeedNode.setEnvironment(environment);
    UserThread.execute(() -> seedNode = new SeedNode());
    Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
        if (throwable.getCause() != null && throwable.getCause().getCause() != null && throwable.getCause().getCause() instanceof BlockStoreException) {
            log.error(throwable.getMessage());
        } else {
            log.error("Uncaught Exception from thread " + Thread.currentThread().getName());
            log.error("throwableMessage= " + throwable.getMessage());
            log.error("throwableClass= " + throwable.getClass());
            log.error("Stack trace:\n" + ExceptionUtils.getStackTrace(throwable));
            throwable.printStackTrace();
            log.error("We shut down the app because an unhandled error occurred");
            // We don't use the restart as in case of OutOfMemory errors the restart might fail as well
            // The run loop will restart the node anyway...
            System.exit(EXIT_FAILURE);
        }
    };
    Thread.setDefaultUncaughtExceptionHandler(handler);
    Thread.currentThread().setUncaughtExceptionHandler(handler);
    String maxMemoryOption = environment.getProperty(AppOptionKeys.MAX_MEMORY);
    if (maxMemoryOption != null && !maxMemoryOption.isEmpty()) {
        try {
            maxMemory = Integer.parseInt(maxMemoryOption);
        } catch (Throwable t) {
            log.error(t.getMessage());
        }
    }
    UserThread.runPeriodically(() -> {
        Profiler.printSystemLoad(log);
        long usedMemoryInMB = Profiler.getUsedMemoryInMB();
        if (!stopped) {
            if (usedMemoryInMB > (maxMemory - 100)) {
                log.warn("\n\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n" + "We are over our memory warn limit and call the GC. usedMemoryInMB: {}" + "\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\n", usedMemoryInMB);
                System.gc();
                usedMemoryInMB = Profiler.getUsedMemoryInMB();
                Profiler.printSystemLoad(log);
            }
            final long finalUsedMemoryInMB = usedMemoryInMB;
            UserThread.runAfter(() -> {
                if (finalUsedMemoryInMB > maxMemory)
                    restart(environment);
            }, 1);
        }
    }, CHECK_MEMORY_PERIOD_SEC);
    while (true) {
        try {
            Thread.sleep(Long.MAX_VALUE);
        } catch (InterruptedException ignore) {
        }
    }
}
Also used : ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) Logger(org.slf4j.Logger) UserThread(io.bitsquare.common.UserThread) LoggerFactory(org.slf4j.LoggerFactory) IOException(java.io.IOException) DEFAULT_USER_DATA_DIR(io.bitsquare.app.BitsquareEnvironment.DEFAULT_USER_DATA_DIR) Executors(java.util.concurrent.Executors) AppOptionKeys(io.bitsquare.app.AppOptionKeys) OptionException(joptsimple.OptionException) BitsquareEnvironment(io.bitsquare.app.BitsquareEnvironment) BitsquareExecutable(io.bitsquare.app.BitsquareExecutable) OptionParser(joptsimple.OptionParser) RestartUtil(io.bitsquare.common.util.RestartUtil) Profiler(io.bitsquare.common.util.Profiler) BlockStoreException(org.bitcoinj.store.BlockStoreException) ThreadFactory(java.util.concurrent.ThreadFactory) OptionSet(joptsimple.OptionSet) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) DEFAULT_APP_NAME(io.bitsquare.app.BitsquareEnvironment.DEFAULT_APP_NAME) BlockStoreException(org.bitcoinj.store.BlockStoreException) BitsquareEnvironment(io.bitsquare.app.BitsquareEnvironment) UserThread(io.bitsquare.common.UserThread)

Example 2 with ExceptionUtils.getStackTrace

use of org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace in project metron by apache.

the class MultiIndexDao method getLatest.

@Override
public Document getLatest(final String guid, String sensorType) throws IOException {
    Document ret = null;
    List<DocumentContainer> output = indices.parallelStream().map(dao -> {
        try {
            return new DocumentContainer(dao.getLatest(guid, sensorType));
        } catch (Throwable e) {
            return new DocumentContainer(e);
        }
    }).collect(Collectors.toList());
    List<String> error = new ArrayList<>();
    for (DocumentContainer dc : output) {
        if (dc.getException().isPresent()) {
            Throwable e = dc.getException().get();
            error.add(e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e));
        } else {
            if (dc.getDocument().isPresent()) {
                Document d = dc.getDocument().get();
                if (ret == null || ret.getTimestamp() < d.getTimestamp()) {
                    ret = d;
                }
            }
        }
    }
    if (error.size() > 0) {
        throw new IOException(Joiner.on("\n").join(error));
    }
    return ret;
}
Also used : InvalidSearchException(org.apache.metron.indexing.dao.search.InvalidSearchException) Document(org.apache.metron.indexing.dao.update.Document) Iterables(com.google.common.collect.Iterables) FieldType(org.apache.metron.indexing.dao.search.FieldType) GroupRequest(org.apache.metron.indexing.dao.search.GroupRequest) IOException(java.io.IOException) GetRequest(org.apache.metron.indexing.dao.search.GetRequest) SearchRequest(org.apache.metron.indexing.dao.search.SearchRequest) Function(java.util.function.Function) Collectors(java.util.stream.Collectors) GroupResponse(org.apache.metron.indexing.dao.search.GroupResponse) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) Optional(java.util.Optional) Collections(java.util.Collections) Joiner(com.google.common.base.Joiner) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) SearchResponse(org.apache.metron.indexing.dao.search.SearchResponse) ArrayList(java.util.ArrayList) IOException(java.io.IOException) Document(org.apache.metron.indexing.dao.update.Document)

Example 3 with ExceptionUtils.getStackTrace

use of org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace in project bitsquare by bitsquare.

the class BitsquareApp method start.

@Override
public void start(Stage stage) throws IOException {
    BitsquareApp.primaryStage = stage;
    String logPath = Paths.get(env.getProperty(AppOptionKeys.APP_DATA_DIR_KEY), "bitsquare").toString();
    Log.setup(logPath);
    log.info("Log files under: " + logPath);
    Version.printVersion();
    Utilities.printSysInfo();
    Log.setLevel(Level.toLevel(env.getRequiredProperty(CommonOptionKeys.LOG_LEVEL_KEY)));
    UserThread.setExecutor(Platform::runLater);
    UserThread.setTimerClass(UITimer.class);
    shutDownHandler = this::stop;
    // setup UncaughtExceptionHandler
    Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
        // Might come from another thread 
        if (throwable.getCause() != null && throwable.getCause().getCause() != null && throwable.getCause().getCause() instanceof BlockStoreException) {
            log.error(throwable.getMessage());
        } else if (throwable instanceof ClassCastException && "sun.awt.image.BufImgSurfaceData cannot be cast to sun.java2d.xr.XRSurfaceData".equals(throwable.getMessage())) {
            log.warn(throwable.getMessage());
        } else {
            log.error("Uncaught Exception from thread " + Thread.currentThread().getName());
            log.error("throwableMessage= " + throwable.getMessage());
            log.error("throwableClass= " + throwable.getClass());
            log.error("Stack trace:\n" + ExceptionUtils.getStackTrace(throwable));
            throwable.printStackTrace();
            UserThread.execute(() -> showErrorPopup(throwable, false));
        }
    };
    Thread.setDefaultUncaughtExceptionHandler(handler);
    Thread.currentThread().setUncaughtExceptionHandler(handler);
    try {
        Utilities.checkCryptoPolicySetup();
    } catch (NoSuchAlgorithmException | LimitedKeyStrengthException e) {
        e.printStackTrace();
        UserThread.execute(() -> showErrorPopup(e, true));
    }
    Security.addProvider(new BouncyCastleProvider());
    try {
        // Guice
        bitsquareAppModule = new BitsquareAppModule(env, primaryStage);
        injector = Guice.createInjector(bitsquareAppModule);
        injector.getInstance(InjectorViewFactory.class).setInjector(injector);
        Version.setBtcNetworkId(injector.getInstance(BitsquareEnvironment.class).getBitcoinNetwork().ordinal());
        if (Utilities.isLinux())
            System.setProperty("prism.lcdtext", "false");
        Storage.setDatabaseCorruptionHandler((String fileName) -> {
            corruptedDatabaseFiles.add(fileName);
            if (mainView != null)
                mainView.setPersistedFilesCorrupted(corruptedDatabaseFiles);
        });
        // load the main view and create the main scene
        CachingViewLoader viewLoader = injector.getInstance(CachingViewLoader.class);
        mainView = (MainView) viewLoader.load(MainView.class);
        mainView.setPersistedFilesCorrupted(corruptedDatabaseFiles);
        /* Storage.setDatabaseCorruptionHandler((String fileName) -> {
                corruptedDatabaseFiles.add(fileName);
                if (mainView != null)
                    mainView.setPersistedFilesCorrupted(corruptedDatabaseFiles);
            });*/
        //740
        scene = new Scene(mainView.getRoot(), 1200, 700);
        Font.loadFont(getClass().getResource("/fonts/Verdana.ttf").toExternalForm(), 13);
        Font.loadFont(getClass().getResource("/fonts/VerdanaBold.ttf").toExternalForm(), 13);
        Font.loadFont(getClass().getResource("/fonts/VerdanaItalic.ttf").toExternalForm(), 13);
        Font.loadFont(getClass().getResource("/fonts/VerdanaBoldItalic.ttf").toExternalForm(), 13);
        scene.getStylesheets().setAll("/io/bitsquare/gui/bitsquare.css", "/io/bitsquare/gui/images.css", "/io/bitsquare/gui/CandleStickChart.css");
        // configure the system tray
        SystemTray.create(primaryStage, shutDownHandler);
        primaryStage.setOnCloseRequest(event -> {
            event.consume();
            stop();
        });
        scene.addEventHandler(KeyEvent.KEY_RELEASED, keyEvent -> {
            if (new KeyCodeCombination(KeyCode.W, KeyCombination.SHORTCUT_DOWN).match(keyEvent) || new KeyCodeCombination(KeyCode.W, KeyCombination.CONTROL_DOWN).match(keyEvent)) {
                stop();
            } else if (new KeyCodeCombination(KeyCode.Q, KeyCombination.SHORTCUT_DOWN).match(keyEvent) || new KeyCodeCombination(KeyCode.Q, KeyCombination.CONTROL_DOWN).match(keyEvent)) {
                stop();
            } else if (new KeyCodeCombination(KeyCode.E, KeyCombination.SHORTCUT_DOWN).match(keyEvent) || new KeyCodeCombination(KeyCode.E, KeyCombination.CONTROL_DOWN).match(keyEvent)) {
                showEmptyWalletPopup();
            } else if (new KeyCodeCombination(KeyCode.M, KeyCombination.ALT_DOWN).match(keyEvent)) {
                showSendAlertMessagePopup();
            } else if (new KeyCodeCombination(KeyCode.F, KeyCombination.ALT_DOWN).match(keyEvent)) {
                showFilterPopup();
            } else if (new KeyCodeCombination(KeyCode.F, KeyCombination.ALT_DOWN).match(keyEvent)) {
                showFPSWindow();
            } else if (new KeyCodeCombination(KeyCode.J, KeyCombination.ALT_DOWN).match(keyEvent)) {
                WalletService walletService = injector.getInstance(WalletService.class);
                if (walletService.getWallet() != null)
                    new ShowWalletDataWindow(walletService).information("Wallet raw data").show();
                else
                    new Popup<>().warning("The wallet is not initialized yet").show();
            } else if (new KeyCodeCombination(KeyCode.G, KeyCombination.ALT_DOWN).match(keyEvent)) {
                TradeWalletService tradeWalletService = injector.getInstance(TradeWalletService.class);
                WalletService walletService = injector.getInstance(WalletService.class);
                if (walletService.getWallet() != null)
                    new SpendFromDepositTxWindow(tradeWalletService).information("Emergency wallet tool").show();
                else
                    new Popup<>().warning("The wallet is not initialized yet").show();
            } else if (DevFlags.DEV_MODE && new KeyCodeCombination(KeyCode.D, KeyCombination.SHORTCUT_DOWN).match(keyEvent)) {
                showDebugWindow();
            }
        });
        // configure the primary stage
        primaryStage.setTitle(env.getRequiredProperty(APP_NAME_KEY));
        primaryStage.setScene(scene);
        // 1190
        primaryStage.setMinWidth(1000);
        primaryStage.setMinHeight(620);
        // on windows the title icon is also used as task bar icon in a larger size
        // on Linux no title icon is supported but also a large task bar icon is derived from that title icon
        String iconPath;
        if (Utilities.isOSX())
            iconPath = ImageUtil.isRetina() ? "/images/window_icon@2x.png" : "/images/window_icon.png";
        else if (Utilities.isWindows())
            iconPath = "/images/task_bar_icon_windows.png";
        else
            iconPath = "/images/task_bar_icon_linux.png";
        primaryStage.getIcons().add(new Image(getClass().getResourceAsStream(iconPath)));
        // make the UI visible
        primaryStage.show();
        if (!Utilities.isCorrectOSArchitecture()) {
            String osArchitecture = Utilities.getOSArchitecture();
            // We don't force a shutdown as the osArchitecture might in strange cases return a wrong value.
            // Needs at least more testing on different machines...
            new Popup<>().warning("You probably have the wrong Bitsquare version for this computer.\n" + "Your computer's architecture is: " + osArchitecture + ".\n" + "The Bitsquare binary you installed is: " + Utilities.getJVMArchitecture() + ".\n" + "Please shut down and re-install the correct version (" + osArchitecture + ").").show();
        }
        UserThread.runPeriodically(() -> Profiler.printSystemLoad(log), LOG_MEMORY_PERIOD_MIN, TimeUnit.MINUTES);
    } catch (Throwable throwable) {
        showErrorPopup(throwable, false);
    }
}
Also used : StageStyle(javafx.stage.StageStyle) Popup(io.bitsquare.gui.main.overlays.popups.Popup) LoggerFactory(org.slf4j.LoggerFactory) Security(java.security.Security) View(io.bitsquare.gui.common.view.View) StackPane(javafx.scene.layout.StackPane) KeyCombination(javafx.scene.input.KeyCombination) Application(javafx.application.Application) Parent(javafx.scene.Parent) UITimer(io.bitsquare.gui.common.UITimer) TradeWalletService(io.bitsquare.btc.TradeWalletService) ResultHandler(io.bitsquare.common.handlers.ResultHandler) BlockStoreException(org.bitcoinj.store.BlockStoreException) Pane(javafx.scene.layout.Pane) Font(javafx.scene.text.Font) LimitedKeyStrengthException(io.bitsquare.common.util.LimitedKeyStrengthException) FilterManager(io.bitsquare.filter.FilterManager) KeyEvent(javafx.scene.input.KeyEvent) InjectorViewFactory(io.bitsquare.gui.common.view.guice.InjectorViewFactory) Platform(javafx.application.Platform) List(java.util.List) io.bitsquare.gui.main.overlays.windows(io.bitsquare.gui.main.overlays.windows) Logger(ch.qos.logback.classic.Logger) MainViewModel(io.bitsquare.gui.main.MainViewModel) Environment(org.springframework.core.env.Environment) Dialogs(org.controlsfx.dialog.Dialogs) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) EventStreams(org.reactfx.EventStreams) CommonOptionKeys(io.bitsquare.common.CommonOptionKeys) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) Scene(javafx.scene.Scene) MainView(io.bitsquare.gui.main.MainView) DebugView(io.bitsquare.gui.main.debug.DebugView) P2PService(io.bitsquare.p2p.P2PService) ArrayList(java.util.ArrayList) TradeManager(io.bitsquare.trade.TradeManager) CachingViewLoader(io.bitsquare.gui.common.view.CachingViewLoader) WalletService(io.bitsquare.btc.WalletService) SystemTray(io.bitsquare.gui.SystemTray) APP_NAME_KEY(io.bitsquare.app.AppOptionKeys.APP_NAME_KEY) KeyCode(javafx.scene.input.KeyCode) Modality(javafx.stage.Modality) Utilities(io.bitsquare.common.util.Utilities) Label(javafx.scene.control.Label) UserThread(io.bitsquare.common.UserThread) ImageUtil(io.bitsquare.gui.util.ImageUtil) IOException(java.io.IOException) ViewLoader(io.bitsquare.gui.common.view.ViewLoader) BouncyCastleProvider(org.bouncycastle.jce.provider.BouncyCastleProvider) Injector(com.google.inject.Injector) KeyCodeCombination(javafx.scene.input.KeyCodeCombination) TimeUnit(java.util.concurrent.TimeUnit) Level(ch.qos.logback.classic.Level) Stage(javafx.stage.Stage) Paths(java.nio.file.Paths) OpenOfferManager(io.bitsquare.trade.offer.OpenOfferManager) ArbitratorManager(io.bitsquare.arbitration.ArbitratorManager) Guice(com.google.inject.Guice) Profiler(io.bitsquare.common.util.Profiler) Storage(io.bitsquare.storage.Storage) Image(javafx.scene.image.Image) AlertManager(io.bitsquare.alert.AlertManager) Platform(javafx.application.Platform) TradeWalletService(io.bitsquare.btc.TradeWalletService) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) Image(javafx.scene.image.Image) TradeWalletService(io.bitsquare.btc.TradeWalletService) WalletService(io.bitsquare.btc.WalletService) CachingViewLoader(io.bitsquare.gui.common.view.CachingViewLoader) LimitedKeyStrengthException(io.bitsquare.common.util.LimitedKeyStrengthException) Popup(io.bitsquare.gui.main.overlays.popups.Popup) InjectorViewFactory(io.bitsquare.gui.common.view.guice.InjectorViewFactory) BouncyCastleProvider(org.bouncycastle.jce.provider.BouncyCastleProvider) BlockStoreException(org.bitcoinj.store.BlockStoreException) KeyCodeCombination(javafx.scene.input.KeyCodeCombination) Scene(javafx.scene.Scene) UserThread(io.bitsquare.common.UserThread)

Example 4 with ExceptionUtils.getStackTrace

use of org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace in project page-factory-2 by sbtqa.

the class JunitReporter method handleStep.

public static Object handleStep(ProceedingJoinPoint joinPoint) throws Throwable {
    // FIXME: need to get another way to filter junit only steps cuz getStackTrace is very hard
    boolean isFromCucumber = Arrays.stream(Thread.currentThread().getStackTrace()).anyMatch(stackTraceElement -> stackTraceElement.getClassName().matches("ru\\.sbtqa\\.tag\\.stepdefs\\.[en|ru]\\..*"));
    if (isFromCucumber) {
        return joinPoint.proceed();
    } else {
        boolean isTestCaseStarted = Allure.getLifecycle().getCurrentTestCase().isPresent();
        if (!isTestCaseStarted) {
            String testCaseId = MD5.hash(joinPoint.getSignature().toShortString());
            String testResultUid = MD5.hash(joinPoint.toLongString());
            Allure.getLifecycle().scheduleTestCase(new TestResult().setTestCaseId(testCaseId).setUuid(testResultUid));
            Allure.getLifecycle().startTestCase(testResultUid);
        }
        Object[] args = normalizeArgs(joinPoint.getArgs());
        String methodName = joinPoint.getSignature().getName();
        // I18n contains template for steps as <methodName><dot><argsCount>. For example: fill.2
        String methodNameWithArgsCount = methodName + "." + args.length;
        String stepUid = createUid(joinPoint);
        String stepNameI18n = getStepNameI18n(joinPoint, methodNameWithArgsCount);
        // if step has i18n template - substitute args to it
        String stepName = String.format((stepNameI18n.equals(methodNameWithArgsCount) ? methodName : stepNameI18n), args);
        Allure.getLifecycle().startStep(stepUid, new StepResult().setName(stepName));
        System.out.println("\t * " + stepName);
        try {
            Object r = joinPoint.proceed();
            Allure.getLifecycle().updateStep(stepUid, stepResult -> stepResult.setStatus(Status.PASSED));
            return r;
        } catch (Throwable t) {
            Allure.getLifecycle().updateStep(stepUid, stepResult -> stepResult.setStatus(Status.FAILED).setStatusDetails(new StatusDetails().setTrace(ExceptionUtils.getStackTrace(t)).setMessage(t.getMessage())));
            throw t;
        } finally {
            attachParameters(methodName, args, createUid(joinPoint), stepName);
            Allure.getLifecycle().stopStep(stepUid);
        }
    }
}
Also used : Status(io.qameta.allure.model.Status) MD5(ru.sbtqa.tag.pagefactory.utils.MD5) Arrays(java.util.Arrays) I18N(ru.sbtqa.tag.qautils.i18n.I18N) TestResult(io.qameta.allure.model.TestResult) PageEntry(ru.sbtqa.tag.pagefactory.annotations.PageEntry) Page(ru.sbtqa.tag.pagefactory.Page) Endpoint(ru.sbtqa.tag.pagefactory.annotations.rest.Endpoint) StatusDetails(io.qameta.allure.model.StatusDetails) Allure(io.qameta.allure.Allure) Locale(java.util.Locale) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) ApiEndpoint(ru.sbtqa.tag.pagefactory.ApiEndpoint) StepResult(io.qameta.allure.model.StepResult) ProceedingJoinPoint(org.aspectj.lang.ProceedingJoinPoint) Configuration(ru.sbtqa.tag.pagefactory.properties.Configuration) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) StatusDetails(io.qameta.allure.model.StatusDetails) TestResult(io.qameta.allure.model.TestResult) StepResult(io.qameta.allure.model.StepResult)

Example 5 with ExceptionUtils.getStackTrace

use of org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace in project hive by apache.

the class TezJobMonitor method monitorExecution.

public int monitorExecution() {
    boolean done = false;
    boolean success = false;
    int failedCounter = 0;
    final StopWatch failureTimer = new StopWatch();
    int rc = 0;
    DAGStatus status = null;
    Map<String, Progress> vertexProgressMap = null;
    long monitorStartTime = System.currentTimeMillis();
    synchronized (shutdownList) {
        shutdownList.add(dagClient);
    }
    perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_DAG);
    perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING);
    DAGStatus.State lastState = null;
    boolean running = false;
    long checkInterval = HiveConf.getTimeVar(hiveConf, HiveConf.ConfVars.TEZ_DAG_STATUS_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
    WmContext wmContext = null;
    while (true) {
        try {
            if (context != null) {
                context.checkHeartbeaterLockException();
            }
            wmContext = context.getWmContext();
            EnumSet<StatusGetOpts> opts = null;
            if (wmContext != null) {
                Set<String> desiredCounters = wmContext.getSubscribedCounters();
                if (desiredCounters != null && !desiredCounters.isEmpty()) {
                    opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
                }
            }
            status = dagClient.getDAGStatus(opts, checkInterval);
            vertexProgressMap = status.getVertexProgress();
            List<String> vertexNames = vertexProgressMap.keySet().stream().map(k -> k.replaceAll(" ", "_")).collect(Collectors.toList());
            if (wmContext != null) {
                Set<String> desiredCounters = wmContext.getSubscribedCounters();
                TezCounters dagCounters = status.getDAGCounters();
                // if initial counters exists, merge it with dag counters to get aggregated view
                TezCounters mergedCounters = counters == null ? dagCounters : Utils.mergeTezCounters(dagCounters, counters);
                if (mergedCounters != null && desiredCounters != null && !desiredCounters.isEmpty()) {
                    Map<String, Long> currentCounters = getCounterValues(mergedCounters, vertexNames, vertexProgressMap, desiredCounters, done);
                    LOG.debug("Requested DAG status. checkInterval: {}. currentCounters: {}", checkInterval, currentCounters);
                    wmContext.setCurrentCounters(currentCounters);
                }
            }
            DAGStatus.State state = status.getState();
            // AM is responsive again (recovery?)
            failedCounter = 0;
            failureTimer.reset();
            if (state != lastState || state == RUNNING) {
                lastState = state;
                switch(state) {
                    case SUBMITTED:
                        console.printInfo("Status: Submitted");
                        break;
                    case INITING:
                        console.printInfo("Status: Initializing");
                        this.executionStartTime = System.currentTimeMillis();
                        break;
                    case RUNNING:
                        if (!running) {
                            perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING);
                            console.printInfo("Status: Running (" + dagClient.getExecutionContext() + ")\n");
                            this.executionStartTime = System.currentTimeMillis();
                            running = true;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        break;
                    case SUCCEEDED:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        success = true;
                        running = false;
                        done = true;
                        break;
                    case KILLED:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        console.printInfo("Status: Killed");
                        running = false;
                        done = true;
                        rc = 1;
                        break;
                    case FAILED:
                    case ERROR:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        console.printError("Status: Failed");
                        running = false;
                        done = true;
                        rc = 2;
                        break;
                }
            }
            if (wmContext != null && done) {
                wmContext.setQueryCompleted(true);
            }
        } catch (Exception e) {
            console.printInfo("Exception: " + e.getMessage());
            boolean isInterrupted = hasInterruptedException(e);
            if (failedCounter == 0) {
                failureTimer.reset();
                failureTimer.start();
            }
            if (isInterrupted || (++failedCounter >= MAX_RETRY_FAILURES && failureTimer.now(TimeUnit.MILLISECONDS) > MAX_RETRY_INTERVAL)) {
                try {
                    if (isInterrupted) {
                        console.printInfo("Killing DAG...");
                    } else {
                        console.printInfo(String.format("Killing DAG... after %d seconds", failureTimer.now(TimeUnit.SECONDS)));
                    }
                    dagClient.tryKillDAG();
                } catch (IOException | TezException tezException) {
                // best effort
                }
                console.printError("Execution has failed. stack trace: " + ExceptionUtils.getStackTrace(e));
                rc = 1;
                done = true;
            } else {
                console.printInfo("Retrying...");
            }
            if (wmContext != null && done) {
                wmContext.setQueryCompleted(true);
            }
        } finally {
            if (done) {
                if (wmContext != null && done) {
                    wmContext.setQueryCompleted(true);
                }
                if (rc != 0 && status != null) {
                    for (String diag : status.getDiagnostics()) {
                        console.printError(diag);
                        diagnostics.append(diag);
                    }
                }
                synchronized (shutdownList) {
                    shutdownList.remove(dagClient);
                }
                break;
            }
        }
    }
    perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.TEZ_RUN_DAG);
    printSummary(success, vertexProgressMap);
    return rc;
}
Also used : DAGClient(org.apache.tez.dag.api.client.DAGClient) StatusGetOpts(org.apache.tez.dag.api.client.StatusGetOpts) TezCounter(org.apache.tez.common.counters.TezCounter) ConfVars(org.apache.hadoop.hive.conf.HiveConf.ConfVars) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) InterruptedIOException(java.io.InterruptedIOException) TimeCounterLimit(org.apache.hadoop.hive.ql.wm.TimeCounterLimit) VertexCounterLimit(org.apache.hadoop.hive.ql.wm.VertexCounterLimit) Utilities(org.apache.hadoop.hive.ql.exec.Utilities) DAGStatus(org.apache.tez.dag.api.client.DAGStatus) ProgressMonitor(org.apache.hadoop.hive.common.log.ProgressMonitor) Map(java.util.Map) InPlaceUpdate(org.apache.hadoop.hive.common.log.InPlaceUpdate) Context(org.apache.hadoop.hive.ql.Context) BaseWork(org.apache.hadoop.hive.ql.plan.BaseWork) CounterGroup(org.apache.tez.common.counters.CounterGroup) LinkedList(java.util.LinkedList) EnumSet(java.util.EnumSet) PerfLogger(org.apache.hadoop.hive.ql.log.PerfLogger) Progress(org.apache.tez.dag.api.client.Progress) Logger(org.slf4j.Logger) StringWriter(java.io.StringWriter) HiveConf(org.apache.hadoop.hive.conf.HiveConf) Set(java.util.Set) StopWatch(org.apache.tez.util.StopWatch) IOException(java.io.IOException) DAG(org.apache.tez.dag.api.DAG) Collectors(java.util.stream.Collectors) SessionState(org.apache.hadoop.hive.ql.session.SessionState) RUNNING(org.apache.tez.dag.api.client.DAGStatus.State.RUNNING) ShutdownHookManager(org.apache.hive.common.util.ShutdownHookManager) LogHelper(org.apache.hadoop.hive.ql.session.SessionState.LogHelper) TezException(org.apache.tez.dag.api.TezException) TimeUnit(java.util.concurrent.TimeUnit) TezCounters(org.apache.tez.common.counters.TezCounters) List(java.util.List) WmContext(org.apache.hadoop.hive.ql.wm.WmContext) Utils(org.apache.hadoop.hive.ql.exec.tez.Utils) Preconditions(com.google.common.base.Preconditions) TezSessionPoolManager(org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) Progress(org.apache.tez.dag.api.client.Progress) WmContext(org.apache.hadoop.hive.ql.wm.WmContext) TezCounters(org.apache.tez.common.counters.TezCounters) InterruptedIOException(java.io.InterruptedIOException) IOException(java.io.IOException) TezException(org.apache.tez.dag.api.TezException) StopWatch(org.apache.tez.util.StopWatch) StatusGetOpts(org.apache.tez.dag.api.client.StatusGetOpts) DAGStatus(org.apache.tez.dag.api.client.DAGStatus)

Aggregations

ExceptionUtils (org.apache.commons.lang3.exception.ExceptionUtils)16 IOException (java.io.IOException)11 List (java.util.List)11 LoggerFactory (org.slf4j.LoggerFactory)9 Map (java.util.Map)7 ArrayList (java.util.ArrayList)6 TimeUnit (java.util.concurrent.TimeUnit)6 BlockStoreException (org.bitcoinj.store.BlockStoreException)6 Logger (org.slf4j.Logger)6 Paths (java.nio.file.Paths)4 Arrays (java.util.Arrays)4 Executors (java.util.concurrent.Executors)4 Level (ch.qos.logback.classic.Level)3 Logger (ch.qos.logback.classic.Logger)3 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)3 Guice (com.google.inject.Guice)3 Injector (com.google.inject.Injector)3 URI (java.net.URI)3 HashMap (java.util.HashMap)3 Optional (java.util.Optional)3