Examples with IClusterDetail com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail used on opensource projects

Example 6 with IClusterDetail

use of com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail in project azure-tools-for-java by Microsoft.

the class HDInsightRootModuleImpl method refreshWithoutAsync.

@Override
public void refreshWithoutAsync() {
    synchronized (this) {
        removeAllChildNodes();
        clusterDetailList = ClusterManagerEx.getInstance().getClusterDetailsWithoutAsync(getProject());
        if (clusterDetailList != null) {
            for (IClusterDetail clusterDetail : clusterDetailList) {
                addChildNode(new ClusterNode(this, clusterDetail));
            }
        }
    }
}

Example 7 with IClusterDetail

use of com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail in project azure-tools-for-java by Microsoft.

the class JobUtils method openSparkEventLog.

public void openSparkEventLog(String uuid, String applicationId) {
    IClusterDetail clusterDetail = JobViewManager.getCluster(uuid);
    String path = StringHelper.concat(HDInsightLoader.getHDInsightHelper().getPluginRootPath(), File.separator, SPARK_EVENT_LOG_FOLDER_NAME, File.separator, applicationId);
    File downloadFile = new File(path, Event_LOG_FILE_NAME);
    File file = new File(path);
    if (!file.exists() || !downloadFile.exists()) {
        if (!file.exists()) {
            file.mkdirs();
        }
        String restApi = String.format(EVENT_LOG_REST_API, applicationId);
        try {
            HttpEntity entity = SparkRestUtil.getEntity(clusterDetail, restApi);
            FileUtils.copyInputStreamToFile(entity.getContent(), downloadFile);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    openFileExplorer(file.toURI());
}

Example 8 with IClusterDetail

use of com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail in project azure-tools-for-java by Microsoft.

the class JobViewDummyHttpServer method initlize.

public static synchronized void initlize() {
    if (isEnabled) {
        return;
    }
    try {
        server = HttpServer.create(new InetSocketAddress(PORT), 10);
        server.createContext("/clusters/", new HttpHandler() {

            @Override
            public void handle(final HttpExchange httpExchange) throws IOException {
                requestDetail = RequestDetail.getRequestDetail(httpExchange.getRequestURI());
                if (requestDetail == null) {
                    return;
                }
                IClusterDetail clusterDetail = requestDetail.getClusterDetail();
                final String queryUrl = requestDetail.getQueryUrl();
                if (requestDetail.getApiType() == RequestDetail.APIType.YarnHistory) {
                    TaskExecutor.submit(new YarnHistoryTask(clusterDetail, queryUrl, new HttpFutureCallback(httpExchange) {

                        @Override
                        public void onSuccess(String str) {
                            httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
                            try {
                                // work around of get job result
                                //TODO: get job result by REST API
                                Document doc = Jsoup.parse(str);
                                Elements contentElements = doc.getElementsByClass("content");
                                if (contentElements.size() == 1) {
                                    Elements elements = contentElements.get(0).getElementsByTag("pre");
                                    if (elements.size() == 1) {
                                        str = elements.get(0).html();
                                    }
                                }
                                httpExchange.sendResponseHeaders(200, str.length());
                                OutputStream stream = httpExchange.getResponseBody();
                                stream.write(str.getBytes());
                                stream.close();
                            } catch (IOException e) {
                                int a = 1;
                            //                                    LOGGER.error("Get job history error", e);
                            }
                        }
                    }));
                } else if (requestDetail.getApiType() == RequestDetail.APIType.LivyBatchesRest) {
                    TaskExecutor.submit(new LivyTask(clusterDetail, queryUrl, new HttpFutureCallback(httpExchange) {

                        @Override
                        public void onSuccess(String str) {
                            httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
                            try {
                                String applicationId = requestDetail.getProperty("applicationId");
                                if (applicationId != null) {
                                    str = JobUtils.getJobInformation(str, applicationId);
                                }
                                httpExchange.sendResponseHeaders(200, str.length());
                                OutputStream stream = httpExchange.getResponseBody();
                                stream.write(str.getBytes());
                                stream.close();
                            } catch (IOException e) {
                            //                                    LOGGER.error("Get job history error", e);
                            }
                        }
                    }));
                } else if (requestDetail.getApiType() == RequestDetail.APIType.MultiTask) {
                    TaskExecutor.submit(new MultiRestTask(clusterDetail, requestDetail.getQueryUrls(), new MultiHttpFutureCallback(httpExchange) {

                        public void onSuccess(List<String> strs) {
                            httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
                            try {
                                String str = tasksDetailsConvert(strs);
                                httpExchange.sendResponseHeaders(200, str.length());
                                OutputStream stream = httpExchange.getResponseBody();
                                stream.write(str.getBytes());
                                stream.close();
                            } catch (IOException e) {
                            //                                    LOGGER.error("Get job history error", e);
                            }
                        }
                    }));
                } else {
                    TaskExecutor.submit(new RestTask(clusterDetail, queryUrl, new HttpFutureCallback(httpExchange) {

                        @Override
                        public void onSuccess(@NotNull String str) {
                            httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
                            try {
                                httpExchange.sendResponseHeaders(200, str.length());
                                OutputStream stream = httpExchange.getResponseBody();
                                stream.write(str.getBytes());
                                stream.close();
                            } catch (IOException e) {
                            //                                    LOGGER.error("Get job history error", e);
                            }
                        }
                    }));
                }
            }
        });
        executorService = Executors.newFixedThreadPool(NO_OF_THREADS);
        server.setExecutor(executorService);
        server.start();
        isEnabled = true;
    } catch (IOException e) {
    //            LOGGER.error("Get job history error", e);
    //            DefaultLoader.getUIHelper().showError(e.getClass().getName(), e.getMessage());
    }
}

Example 9 with IClusterDetail

use of com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail in project azure-tools-for-java by Microsoft.

the class SparkSubmissionExDialog method createContents.

@Override
protected Control createContents(Composite parent) {
    Composite container = new Composite(parent, SWT.NONE);
    GridLayout gridLayout = new GridLayout();
    gridLayout.numColumns = 2;
    container.setLayout(gridLayout);
    GridData gridData = new GridData();
    gridData.widthHint = 550;
    container.setLayoutData(gridData);
    Label clusterListLabel = new Label(container, SWT.LEFT);
    clusterListLabel.setText("Cluster Name:");
    gridLayout = new GridLayout();
    gridLayout.numColumns = 2;
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    Composite composite = new Composite(container, SWT.NONE);
    composite.setLayout(gridLayout);
    composite.setLayoutData(gridData);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    clustersListComboBox = new Combo(composite, SWT.READ_ONLY);
    clustersListComboBox.setLayoutData(gridData);
    clustersListComboBox.setToolTipText("The HDInsight Spark cluster you want to submit your application to. Only Linux cluster is supported.");
    for (IClusterDetail clusterDetail : cachedClusterDetails) {
        //            mapClusterNameToClusterDetail.put(clusterDetail.getName(), clusterDetail);
        clustersListComboBox.add(clusterDetail.getName());
        clustersListComboBox.setData(clusterDetail.getName(), clusterDetail);
    //            if (clusterComboBoxModel.getSize() == 0) {
    //                clusterComboBoxModel.setSelectedItem(clusterDetail.getName());
    //            }
    }
    if (cachedClusterDetails.size() > 0) {
        clustersListComboBox.select(0);
    }
    Button clusterListButton = new Button(composite, SWT.PUSH);
    clusterListButton.setToolTipText("Refresh");
    clusterListButton.setImage(Activator.getImageDescriptor(CommonConst.RefreshIConPath).createImage());
    String tipInfo = "The Artifact you want to use.";
    Label artifactSelectLabel = new Label(container, SWT.LEFT);
    artifactSelectLabel.setText("Select an Artifact to submit");
    artifactSelectLabel.setToolTipText(tipInfo);
    gridData = new GridData();
    gridData.horizontalSpan = 2;
    artifactSelectLabel.setLayoutData(gridData);
    intelliJArtifactRadioButton = new Button(container, SWT.RADIO);
    intelliJArtifactRadioButton.setText("Artifact from Eclipse project:");
    intelliJArtifactRadioButton.setSelection(true);
    intelliJArtifactRadioButton.addSelectionListener(new SelectionListener() {

        @Override
        public void widgetSelected(SelectionEvent arg0) {
            selectedArtifactComboBox.setEnabled(true);
            selectedArtifactTextField.setEnabled(false);
        //				mainClassTextField.setButtonEnabled(true);
        //
        //				setVisibleForFixedErrorMessageLabel(2, false);
        //
        //				if (selectedArtifactComboBox.getItemCount() == 0) {
        //					setVisibleForFixedErrorMessageLabel(2, true);
        //				}
        }

        @Override
        public void widgetDefaultSelected(SelectionEvent arg0) {
        }
    });
    gridData = new GridData();
    gridData.grabExcessHorizontalSpace = true;
    gridData.horizontalAlignment = SWT.FILL;
    selectedArtifactComboBox = new Combo(container, SWT.READ_ONLY);
    selectedArtifactComboBox.setToolTipText(tipInfo);
    selectedArtifactComboBox.setLayoutData(gridData);
    String[] projects = getProjects();
    selectedArtifactComboBox.setItems(projects);
    if (projects.length > 0) {
        selectedArtifactComboBox.select(0);
    }
    localArtifactRadioButton = new Button(container, SWT.RADIO);
    localArtifactRadioButton.setText("Artifact from hard disk:");
    localArtifactRadioButton.addSelectionListener(new SelectionListener() {

        @Override
        public void widgetSelected(SelectionEvent e) {
            selectedArtifactComboBox.setEnabled(false);
            selectedArtifactTextField.setEnabled(true);
        //				mainClassTextField.setButtonEnabled(false);
        //
        //				setVisibleForFixedErrorMessageLabel(1, false);
        //
        //				if (StringHelper.isNullOrWhiteSpace(selectedArtifactTextField.getText())) {
        //					setVisibleForFixedErrorMessageLabel(2, true);
        //				}
        }

        @Override
        public void widgetDefaultSelected(SelectionEvent arg0) {
        // TODO Auto-generated method stub
        }
    });
    gridLayout = new GridLayout();
    gridLayout.numColumns = 2;
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    composite = new Composite(container, SWT.NONE);
    composite.setLayout(gridLayout);
    composite.setLayoutData(gridData);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    selectedArtifactTextField = new Text(composite, SWT.LEFT | SWT.BORDER);
    selectedArtifactTextField.setLayoutData(gridData);
    artifactBrowseButton = new Button(composite, SWT.PUSH);
    artifactBrowseButton.setText("Browse");
    artifactBrowseButton.addSelectionListener(new SelectionAdapter() {

        @Override
        public void widgetSelected(SelectionEvent arg0) {
            FileDialog dialog = new FileDialog(SparkSubmissionExDialog.this.getShell());
            String[] extensions = { "*.jar", "*.JAR" };
            dialog.setFilterExtensions(extensions);
            String file = dialog.open();
            if (file != null) {
                selectedArtifactTextField.setText(file);
            }
        }
    });
    Label sparkMainClassLabel = new Label(container, SWT.LEFT);
    sparkMainClassLabel.setText("Main class name");
    sparkMainClassLabel.setToolTipText("Application's java/spark main class");
    gridLayout = new GridLayout();
    gridLayout.numColumns = 2;
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    composite = new Composite(container, SWT.NONE);
    composite.setLayout(gridLayout);
    composite.setLayoutData(gridData);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    mainClassTextField = new Text(composite, SWT.LEFT | SWT.BORDER);
    mainClassTextField.setLayoutData(gridData);
    Label jobConfigurationLabel = new Label(container, SWT.LEFT);
    jobConfigurationLabel.setText("Job configurations");
    gridData = new GridData();
    gridData.verticalAlignment = SWT.TOP;
    jobConfigurationLabel.setLayoutData(gridData);
    jobConfigurationTable = new Table(container, SWT.BORDER);
    jobConfigurationTable.setHeaderVisible(true);
    jobConfigurationTable.setLinesVisible(true);
    gridData = new GridData();
    gridData.heightHint = 75;
    gridData.horizontalAlignment = SWT.FILL;
    GridLayout gridLayoutTable = new GridLayout();
    gridLayoutTable.numColumns = 2;
    gridLayoutTable.marginRight = 0;
    jobConfigurationTable.setLayout(gridLayoutTable);
    jobConfigurationTable.setLayoutData(gridData);
    TableColumn key = new TableColumn(jobConfigurationTable, SWT.FILL);
    key.setText(COLUMN_NAMES[0]);
    key.setWidth(150);
    TableColumn value = new TableColumn(jobConfigurationTable, SWT.FILL);
    value.setText(COLUMN_NAMES[1]);
    value.setWidth(80);
    tableViewer = new TableViewer(jobConfigurationTable);
    tableViewer.setUseHashlookup(true);
    tableViewer.setColumnProperties(COLUMN_NAMES);
    CellEditor[] editors = new CellEditor[2];
    editors[1] = new TextCellEditor(jobConfigurationTable);
    tableViewer.setCellEditors(editors);
    tableViewer.setContentProvider(new JobConfigurationContentProvider());
    tableViewer.setLabelProvider(new JobConfigurationLabelProvider());
    tableViewer.setCellModifier(new JobConfigurationCellModifier());
    initializeTable();
    Label label = new Label(container, SWT.LEFT);
    label.setText("Command line arguments");
    tipInfo = "Command line arguments used in your main class; multiple arguments should be split by space.";
    label.setToolTipText(tipInfo);
    commandLineTextField = new Text(container, SWT.LEFT | SWT.BORDER);
    commandLineTextField.setToolTipText(tipInfo);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    commandLineTextField.setLayoutData(gridData);
    tipInfo = "Files to be placed on the java classpath; The path needs to be a Azure Blob Storage Path (path started with wasb://); Multiple paths should be split by semicolon (;)";
    Label referencedJarsLabel = new Label(container, SWT.LEFT);
    referencedJarsLabel.setText("Referenced Jars");
    referencedJarsLabel.setToolTipText(tipInfo);
    referencedJarsTextField = new Text(container, SWT.BORDER | SWT.LEFT);
    referencedJarsTextField.setToolTipText(tipInfo);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    referencedJarsTextField.setLayoutData(gridData);
    tipInfo = "Files to be placed in executor working directory. The path needs to be a Azure Blob Storage Path (path started with wasb://); Multiple paths should be split by semicolon (;) ";
    Label referencedFilesLabel = new Label(container, SWT.LEFT);
    referencedFilesLabel.setText("Referenced Files");
    referencedFilesLabel.setToolTipText(tipInfo);
    referencedFilesTextField = new Text(container, SWT.BORDER | SWT.LEFT);
    referencedFilesTextField.setToolTipText(tipInfo);
    gridData = new GridData();
    gridData.horizontalAlignment = SWT.FILL;
    gridData.grabExcessHorizontalSpace = true;
    referencedFilesTextField.setLayoutData(gridData);
    return super.createContents(parent);
}

Example 10 with IClusterDetail

use of com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail in project azure-tools-for-java by Microsoft.

the class SparkBatchJobDebuggerRunner method startDebuggerObservable.

/**
     * Start Spark batch job remote debugging
     *
     * @param environment ID of the {@link Executor} with which the user is trying to run the configuration.
     * @param callback callback when debugger is prepared
     * @param submissionState the submission state from run configuration
     * @param remoteDebugJob the remote Spark job which is listening a port for debugging
     * @return a single Observable with SparkBatchDebugSession instance which is done
     */
protected Single<SparkBatchDebugSession> startDebuggerObservable(@NotNull ExecutionEnvironment environment, @Nullable Callback callback, @NotNull SparkBatchJobSubmissionState submissionState, @NotNull SparkBatchRemoteDebugJob remoteDebugJob) {
    SparkSubmitModel submitModel = submissionState.getSubmitModel();
    IClusterDetail clusterDetail = submitModel.getSelectedClusterDetail();
    return Single.fromEmitter(em -> {
        try {
            SimpleEntry<SparkBatchDebugSession, Integer> sessionPortPair = createSshPortForwardDebugSession(clusterDetail.getConnectionUrl(), submitModel, remoteDebugJob);
            submissionState.setRemoteConnection(new RemoteConnection(true, "localhost", Integer.toString(sessionPortPair.getValue()), false));
            super.execute(environment, callback, submissionState);
            em.onSuccess(sessionPortPair.getKey());
        } catch (Exception ex) {
            em.onError(ex);
        }
    });
}