Examples with KettleException org.pentaho.di.core.exception.KettleException used on opensource projects

Example 1 with KettleException

use of org.pentaho.di.core.exception.KettleException in project pentaho-kettle by pentaho.

the class TextFileCSVImportProgressDialog method doScan.

private String doScan(IProgressMonitor monitor) throws KettleException {
    if (samples > 0) {
        monitor.beginTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile"), samples + 1);
    } else {
        monitor.beginTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile"), 2);
    }
    String line = "";
    long fileLineNumber = 0;
    DecimalFormatSymbols dfs = new DecimalFormatSymbols();
    int nrfields = meta.inputFields.length;
    RowMetaInterface outputRowMeta = new RowMeta();
    meta.getFields(outputRowMeta, null, null, null, transMeta, null, null);
    // Remove the storage meta-data (don't go for lazy conversion during scan)
    for (ValueMetaInterface valueMeta : outputRowMeta.getValueMetaList()) {
        valueMeta.setStorageMetadata(null);
        valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
    }
    RowMetaInterface convertRowMeta = outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
    // How many null values?
    // How many times null value?
    int[] nrnull = new int[nrfields];
    // String info
    // min string
    String[] minstr = new String[nrfields];
    // max string
    String[] maxstr = new String[nrfields];
    // first occ. of string?
    boolean[] firststr = new boolean[nrfields];
    // Date info
    // is the field perhaps a Date?
    boolean[] isDate = new boolean[nrfields];
    // How many date formats work?
    int[] dateFormatCount = new int[nrfields];
    // What are the date formats that
    boolean[][] dateFormat = new boolean[nrfields][Const.getDateFormats().length];
    // work?
    // min date value
    Date[][] minDate = new Date[nrfields][Const.getDateFormats().length];
    // max date value
    Date[][] maxDate = new Date[nrfields][Const.getDateFormats().length];
    // Number info
    // is the field perhaps a Number?
    boolean[] isNumber = new boolean[nrfields];
    // How many number formats work?
    int[] numberFormatCount = new int[nrfields];
    // What are the number format
    boolean[][] numberFormat = new boolean[nrfields][Const.getNumberFormats().length];
    // that work?
    // min number value
    double[][] minValue = new double[nrfields][Const.getDateFormats().length];
    // max number value
    double[][] maxValue = new double[nrfields][Const.getDateFormats().length];
    // remember the precision?
    int[][] numberPrecision = new int[nrfields][Const.getNumberFormats().length];
    // remember the length?
    int[][] numberLength = new int[nrfields][Const.getNumberFormats().length];
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = meta.inputFields[i];
        if (log.isDebug()) {
            debug = "init field #" + i;
        }
        if (replaceMeta) {
            // Clear previous info...
            field.setName(meta.inputFields[i].getName());
            field.setType(meta.inputFields[i].getType());
            field.setFormat("");
            field.setLength(-1);
            field.setPrecision(-1);
            field.setCurrencySymbol(dfs.getCurrencySymbol());
            field.setDecimalSymbol("" + dfs.getDecimalSeparator());
            field.setGroupSymbol("" + dfs.getGroupingSeparator());
            field.setNullString("-");
            field.setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
        }
        nrnull[i] = 0;
        minstr[i] = "";
        maxstr[i] = "";
        firststr[i] = true;
        // Init data guess
        isDate[i] = true;
        for (int j = 0; j < Const.getDateFormats().length; j++) {
            dateFormat[i][j] = true;
            minDate[i][j] = Const.MAX_DATE;
            maxDate[i][j] = Const.MIN_DATE;
        }
        dateFormatCount[i] = Const.getDateFormats().length;
        // Init number guess
        isNumber[i] = true;
        for (int j = 0; j < Const.getNumberFormats().length; j++) {
            numberFormat[i][j] = true;
            minValue[i][j] = Double.MAX_VALUE;
            maxValue[i][j] = -Double.MAX_VALUE;
            numberPrecision[i][j] = -1;
            numberLength[i][j] = -1;
        }
        numberFormatCount[i] = Const.getNumberFormats().length;
    }
    TextFileInputMeta strinfo = (TextFileInputMeta) meta.clone();
    for (int i = 0; i < nrfields; i++) {
        strinfo.inputFields[i].setType(ValueMetaInterface.TYPE_STRING);
    }
    // Sample <samples> rows...
    debug = "get first line";
    StringBuilder lineBuffer = new StringBuilder(256);
    int fileFormatType = meta.getFileFormatTypeNr();
    // If the file has a header we overwrite the first line
    // However, if it doesn't have a header, take a new line
    // 
    line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
    fileLineNumber++;
    int skipped = 1;
    if (meta.content.header) {
        while (line != null && skipped < meta.content.nrHeaderLines) {
            line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
            skipped++;
            fileLineNumber++;
        }
    }
    int linenr = 1;
    List<StringEvaluator> evaluators = new ArrayList<StringEvaluator>();
    // Allocate number and date parsers
    DecimalFormat df2 = (DecimalFormat) NumberFormat.getInstance();
    DecimalFormatSymbols dfs2 = new DecimalFormatSymbols();
    SimpleDateFormat daf2 = new SimpleDateFormat();
    boolean errorFound = false;
    while (!errorFound && line != null && (linenr <= samples || samples == 0) && !monitor.isCanceled()) {
        monitor.subTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningLine", "" + linenr));
        if (samples > 0) {
            monitor.worked(1);
        }
        if (log.isDebug()) {
            debug = "convert line #" + linenr + " to row";
        }
        RowMetaInterface rowMeta = new RowMeta();
        meta.getFields(rowMeta, "stepname", null, null, transMeta, null, null);
        // Remove the storage meta-data (don't go for lazy conversion during scan)
        for (ValueMetaInterface valueMeta : rowMeta.getValueMetaList()) {
            valueMeta.setStorageMetadata(null);
            valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
        }
        String delimiter = transMeta.environmentSubstitute(meta.content.separator);
        String enclosure = transMeta.environmentSubstitute(meta.content.enclosure);
        String escapeCharacter = transMeta.environmentSubstitute(meta.content.escapeCharacter);
        Object[] r = TextFileInputUtils.convertLineToRow(log, new TextFileLine(line, fileLineNumber, null), strinfo, null, 0, outputRowMeta, convertRowMeta, FileInputList.createFilePathList(transMeta, meta.inputFiles.fileName, meta.inputFiles.fileMask, meta.inputFiles.excludeFileMask, meta.inputFiles.fileRequired, meta.inputFiles.includeSubFolderBoolean())[0], rownumber, delimiter, enclosure, escapeCharacter, null, new BaseFileInputAdditionalField(), null, null, false, null, null, null, null, null);
        if (r == null) {
            errorFound = true;
            continue;
        }
        rownumber++;
        for (int i = 0; i < nrfields && i < r.length; i++) {
            StringEvaluator evaluator;
            if (i >= evaluators.size()) {
                evaluator = new StringEvaluator(true);
                evaluators.add(evaluator);
            } else {
                evaluator = evaluators.get(i);
            }
            String string = rowMeta.getString(r, i);
            if (i == 0) {
                System.out.println();
            }
            evaluator.evaluateString(string);
        }
        fileLineNumber++;
        if (r != null) {
            linenr++;
        }
        // Grab another line...
        // 
        line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
    }
    monitor.worked(1);
    monitor.setTaskName(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.AnalyzingResults"));
    // Show information on items using a dialog box
    // 
    StringBuilder message = new StringBuilder();
    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.ResultAfterScanning", "" + (linenr - 1)));
    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.HorizontalLine"));
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = meta.inputFields[i];
        StringEvaluator evaluator = evaluators.get(i);
        List<StringEvaluationResult> evaluationResults = evaluator.getStringEvaluationResults();
        // 
        if (evaluationResults.isEmpty()) {
            field.setType(ValueMetaInterface.TYPE_STRING);
            field.setLength(evaluator.getMaxLength());
        } else {
            StringEvaluationResult result = evaluator.getAdvicedResult();
            if (result != null) {
                // Take the first option we find, list the others below...
                // 
                ValueMetaInterface conversionMeta = result.getConversionMeta();
                field.setType(conversionMeta.getType());
                field.setTrimType(conversionMeta.getTrimType());
                field.setFormat(conversionMeta.getConversionMask());
                field.setDecimalSymbol(conversionMeta.getDecimalSymbol());
                field.setGroupSymbol(conversionMeta.getGroupingSymbol());
                field.setLength(conversionMeta.getLength());
                field.setPrecision(conversionMeta.getPrecision());
                nrnull[i] = result.getNrNull();
                minstr[i] = result.getMin() == null ? "" : result.getMin().toString();
                maxstr[i] = result.getMax() == null ? "" : result.getMax().toString();
            }
        }
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldNumber", "" + (i + 1)));
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldName", field.getName()));
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldType", field.getTypeDesc()));
        switch(field.getType()) {
            case ValueMetaInterface.TYPE_NUMBER:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.EstimatedLength", (field.getLength() < 0 ? "-" : "" + field.getLength())));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.EstimatedPrecision", field.getPrecision() < 0 ? "-" : "" + field.getPrecision()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat", field.getFormat()));
                if (!evaluationResults.isEmpty()) {
                    if (evaluationResults.size() > 1) {
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.WarnNumberFormat"));
                    }
                    for (StringEvaluationResult seResult : evaluationResults) {
                        String mask = seResult.getConversionMeta().getConversionMask();
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat2", mask));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.TrimType", seResult.getConversionMeta().getTrimType()));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberMinValue", seResult.getMin()));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberMaxValue", seResult.getMax()));
                        try {
                            df2.applyPattern(mask);
                            df2.setDecimalFormatSymbols(dfs2);
                            double mn = df2.parse(seResult.getMin().toString()).doubleValue();
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberExample", mask, seResult.getMin(), Double.toString(mn)));
                        } catch (Exception e) {
                            if (log.isDetailed()) {
                                log.logDetailed("This is unexpected: parsing [" + seResult.getMin() + "] with format [" + mask + "] did not work.");
                            }
                        }
                    }
                }
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberNrNullValues", "" + nrnull[i]));
                break;
            case ValueMetaInterface.TYPE_STRING:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMaxLength", "" + field.getLength()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMinValue", minstr[i]));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMaxValue", maxstr[i]));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringNrNullValues", "" + nrnull[i]));
                break;
            case ValueMetaInterface.TYPE_DATE:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMaxLength", field.getLength() < 0 ? "-" : "" + field.getLength()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateFormat", field.getFormat()));
                if (dateFormatCount[i] > 1) {
                    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.WarnDateFormat"));
                }
                if (!Utils.isEmpty(minstr[i])) {
                    for (int x = 0; x < Const.getDateFormats().length; x++) {
                        if (dateFormat[i][x]) {
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateFormat2", Const.getDateFormats()[x]));
                            Date mindate = minDate[i][x];
                            Date maxdate = maxDate[i][x];
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMinValue", mindate.toString()));
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMaxValue", maxdate.toString()));
                            daf2.applyPattern(Const.getDateFormats()[x]);
                            try {
                                Date md = daf2.parse(minstr[i]);
                                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateExample", Const.getDateFormats()[x], minstr[i], md.toString()));
                            } catch (Exception e) {
                                if (log.isDetailed()) {
                                    log.logDetailed("This is unexpected: parsing [" + minstr[i] + "] with format [" + Const.getDateFormats()[x] + "] did not work.");
                                }
                            }
                        }
                    }
                }
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateNrNullValues", "" + nrnull[i]));
                break;
            default:
                break;
        }
        if (nrnull[i] == linenr - 1) {
            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.AllNullValues"));
        }
        message.append(Const.CR);
    }
    monitor.worked(1);
    monitor.done();
    return message.toString();
}

Example 2 with KettleException

use of org.pentaho.di.core.exception.KettleException in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    TextFileInputMeta meta = new TextFileInputMeta();
    getInfo(meta, true);
    // CSV without separator defined
    if (meta.content.fileType.equalsIgnoreCase("CSV") && (meta.content.separator == null || meta.content.separator.isEmpty())) {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "TextFileInput.Exception.NoSeparator"));
        mb.setText(BaseMessages.getString(PKG, "TextFileInputDialog.DialogTitle"));
        mb.open();
        return;
    }
    TextFileInputMeta previousMeta = (TextFileInputMeta) meta.clone();
    FileInputList textFileList = meta.getFileInputList(transMeta);
    InputStream fileInputStream;
    CompressionInputStream inputStream = null;
    StringBuilder lineStringBuilder = new StringBuilder(256);
    int fileFormatType = meta.getFileFormatTypeNr();
    String delimiter = transMeta.environmentSubstitute(meta.content.separator);
    String enclosure = transMeta.environmentSubstitute(meta.content.enclosure);
    String escapeCharacter = transMeta.environmentSubstitute(meta.content.escapeCharacter);
    if (textFileList.nrOfFiles() > 0) {
        int clearFields = meta.content.header ? SWT.YES : SWT.NO;
        int nrInputFields = meta.inputFields.length;
        if (nrInputFields > 0) {
            MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
            mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogMessage"));
            mb.setText(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogTitle"));
            clearFields = mb.open();
            if (clearFields == SWT.CANCEL) {
                return;
            }
        }
        try {
            wFields.table.removeAll();
            FileObject fileObject = textFileList.getFile(0);
            fileInputStream = KettleVFS.getInputStream(fileObject);
            Table table = wFields.table;
            CompressionProvider provider = CompressionProviderFactory.getInstance().createCompressionProviderInstance(meta.content.fileCompression);
            inputStream = provider.createInputStream(fileInputStream);
            InputStreamReader reader;
            if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
                reader = new InputStreamReader(inputStream, meta.getEncoding());
            } else {
                reader = new InputStreamReader(inputStream);
            }
            EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
            // Scan the header-line, determine fields...
            String line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineStringBuilder);
            if (line != null) {
                // Estimate the number of input fields...
                // Chop up the line using the delimiter
                String[] fields = TextFileInputUtils.guessStringsFromLine(transMeta, log, line, meta, delimiter, enclosure, escapeCharacter);
                for (int i = 0; i < fields.length; i++) {
                    String field = fields[i];
                    if (field == null || field.length() == 0 || !meta.content.header) {
                        field = "Field" + (i + 1);
                    } else {
                        // Trim the field
                        field = Const.trim(field);
                        // Replace all spaces & - with underscore _
                        field = Const.replace(field, " ", "_");
                        field = Const.replace(field, "-", "_");
                    }
                    TableItem item = new TableItem(table, SWT.NONE);
                    item.setText(1, field);
                    // The default type is String...
                    item.setText(2, "String");
                }
                wFields.setRowNums();
                wFields.optWidth(true);
                // Copy it...
                getInfo(meta, true);
                // Sample a few lines to determine the correct type of the fields...
                String shellText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogTitle");
                String lineText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogMessage");
                EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
                int samples = end.open();
                if (samples >= 0) {
                    getInfo(meta, true);
                    TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, clearFields == SWT.YES);
                    String message = pd.open();
                    if (message != null) {
                        wFields.removeAll();
                        // OK, what's the result of our search?
                        getData(meta);
                        // 
                        if (clearFields == SWT.NO) {
                            getFieldsData(previousMeta, true);
                            wFields.table.setSelection(previousMeta.inputFields.length, wFields.table.getItemCount() - 1);
                        }
                        wFields.removeEmptyRows();
                        wFields.setRowNums();
                        wFields.optWidth(true);
                        EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogMessage"), message, true);
                        etd.setReadOnly();
                        etd.open();
                    }
                }
            } else {
                MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
                mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.UnableToReadHeaderLine.DialogMessage"));
                mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
                mb.open();
            }
        } catch (IOException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogMessage"), e);
        } catch (KettleException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "TextFileInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
        } finally {
            try {
                if (inputStream != null) {
                    inputStream.close();
                }
            } catch (Exception e) {
            // Ignore errors
            }
        }
    } else {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.NoValidFileFound.DialogMessage"));
        mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
        mb.open();
    }
}

Example 3 with KettleException

use of org.pentaho.di.core.exception.KettleException in project pentaho-kettle by pentaho.

the class KitchenCommandExecutor method execute.

public int execute(String repoName, String noRepo, String username, String trustUser, String password, String dirName, String filename, String jobName, String listJobs, String listDirs, String exportRepo, String initialDir, String listRepos, String listParams, NamedParams params, NamedParams customParams, String[] arguments) throws Throwable {
    getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Kitchen.Log.Starting"));
    Date start = Calendar.getInstance().getTime();
    logDebug("Kitchen.Log.AllocateNewJob");
    Job job = null;
    // In case we use a repository...
    Repository repository = null;
    try {
        if (getMetaStore() == null) {
            setMetaStore(createDefaultMetastore());
        }
        // Read kettle job specified on command-line?
        if (!Utils.isEmpty(repoName) || !Utils.isEmpty(filename)) {
            logDebug("Kitchen.Log.ParsingCommandLine");
            if (!Utils.isEmpty(repoName) && !YES.equalsIgnoreCase(noRepo)) {
                /**
                 * if set, _trust_user_ needs to be considered. See pur-plugin's:
                 *
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepositoryConnector.java#L97-L101
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/WebServiceManager.java#L130-L133
                 */
                if (YES.equalsIgnoreCase(trustUser)) {
                    System.setProperty("pentaho.repository.client.attemptTrust", YES);
                }
                // In case we use a repository...
                // some commands are to load a Trans from the repo; others are merely to print some repo-related information
                RepositoryMeta repositoryMeta = loadRepositoryConnection(repoName, "Kitchen.Log.LoadingRep", "Kitchen.Error.NoRepDefinied", "Kitchen.Log.FindingRep");
                repository = establishRepositoryConnection(repositoryMeta, username, password, RepositoryOperation.EXECUTE_JOB);
                job = executeRepositoryBasedCommand(repository, repositoryMeta, dirName, jobName, listJobs, listDirs);
            }
            // Try to load if from file anyway.
            if (!Utils.isEmpty(filename) && job == null) {
                // Try to load the job from file, even if it failed to load from the repository
                job = executeFilesystemBasedCommand(initialDir, filename);
            }
        } else if (YES.equalsIgnoreCase(listRepos)) {
            // list the repositories placed at repositories.xml
            printRepositories(loadRepositoryInfo("Kitchen.Log.ListRep", "Kitchen.Error.NoRepDefinied"));
        }
    } catch (KettleException e) {
        job = null;
        if (repository != null) {
            repository.disconnect();
        }
        System.out.println(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.StopProcess", e.getMessage()));
    }
    if (job == null) {
        if (!YES.equalsIgnoreCase(listJobs) && !YES.equalsIgnoreCase(listDirs) && !YES.equalsIgnoreCase(listRepos)) {
            System.out.println(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.canNotLoadJob"));
        }
        return CommandExecutorCodes.Kitchen.COULD_NOT_LOAD_JOB.getCode();
    }
    if (!Utils.isEmpty(exportRepo)) {
        try {
            // Export the resources linked to the currently loaded file...
            TopLevelResource topLevelResource = ResourceUtil.serializeResourceExportInterface(exportRepo, job.getJobMeta(), job, repository, getMetaStore());
            String launchFile = topLevelResource.getResourceName();
            String message = ResourceUtil.getExplanation(exportRepo, launchFile, job.getJobMeta());
            System.out.println();
            System.out.println(message);
            // Setting the list parameters option will make kitchen exit below in the parameters section
            listParams = YES;
        } catch (Exception e) {
            System.out.println(Const.getStackTracker(e));
            return CommandExecutorCodes.Kitchen.UNEXPECTED_ERROR.getCode();
        }
    }
    Result result = null;
    int returnCode = CommandExecutorCodes.Kitchen.SUCCESS.getCode();
    try {
        // Set the command line arguments on the job ...
        job.setArguments(arguments != null ? arguments : null);
        job.initializeVariablesFrom(null);
        job.setLogLevel(getLog().getLogLevel());
        job.getJobMeta().setInternalKettleVariables(job);
        job.setRepository(repository);
        job.getJobMeta().setRepository(repository);
        job.getJobMeta().setMetaStore(getMetaStore());
        // Map the command line named parameters to the actual named parameters. Skip for
        // the moment any extra command line parameter not known in the job.
        String[] jobParams = job.getJobMeta().listParameters();
        for (String param : jobParams) {
            String value = params.getParameterValue(param);
            if (value != null) {
                job.getJobMeta().setParameterValue(param, value);
            }
        }
        job.copyParametersFrom(job.getJobMeta());
        // Put the parameters over the already defined variable space. Parameters get priority.
        job.activateParameters();
        // Set custom options in the job extension map as Strings
        for (String optionName : customParams.listParameters()) {
            String optionValue = customParams.getParameterValue(optionName);
            if (optionName != null && optionValue != null) {
                job.getExtensionDataMap().put(optionName, optionValue);
            }
        }
        // List the parameters defined in this job, then simply exit...
        if (YES.equalsIgnoreCase(listParams)) {
            printJobParameters(job);
            // same as the other list options
            return CommandExecutorCodes.Kitchen.COULD_NOT_LOAD_JOB.getCode();
        }
        job.start();
        job.waitUntilFinished();
        // Execute the selected job.
        result = job.getResult();
    } finally {
        if (repository != null) {
            repository.disconnect();
        }
        if (YES.equalsIgnoreCase(trustUser)) {
            // we set it, now we sanitize it
            System.clearProperty("pentaho.repository.client.attemptTrust");
        }
    }
    getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Kitchen.Log.Finished"));
    if (result != null && result.getNrErrors() != 0) {
        getLog().logError(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.FinishedWithErrors"));
        returnCode = CommandExecutorCodes.Kitchen.ERRORS_DURING_PROCESSING.getCode();
    }
    Date stop = Calendar.getInstance().getTime();
    calculateAndPrintElapsedTime(start, stop, "Kitchen.Log.StartStop", "Kitchen.Log.ProcessEndAfter", "Kitchen.Log.ProcessEndAfterLong", "Kitchen.Log.ProcessEndAfterLonger", "Kitchen.Log.ProcessEndAfterLongest");
    return returnCode;
}

Example 4 with KettleException

use of org.pentaho.di.core.exception.KettleException in project pentaho-kettle by pentaho.

the class StepWithMappingMeta method loadMappingMeta.

public static synchronized TransMeta loadMappingMeta(StepWithMappingMeta executorMeta, Repository rep, IMetaStore metaStore, VariableSpace space, boolean share) throws KettleException {
    TransMeta mappingTransMeta = null;
    CurrentDirectoryResolver r = new CurrentDirectoryResolver();
    // send parentVariables = null we don't need it here for resolving resolveCurrentDirectory.
    // Otherwise we destroy child variables and the option "Inherit all variables from the transformation" is enabled always.
    VariableSpace tmpSpace = r.resolveCurrentDirectory(executorMeta.getSpecificationMethod(), null, rep, executorMeta.getParentStepMeta(), executorMeta.getFileName());
    switch(executorMeta.getSpecificationMethod()) {
        case FILENAME:
            String realFilename = tmpSpace.environmentSubstitute(executorMeta.getFileName());
            try {
                // Don't set internal variables: they belong to the parent thread!
                if (rep != null) {
                    // need to try to load from the repository
                    realFilename = r.normalizeSlashes(realFilename);
                    try {
                        String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
                        String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1);
                        RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
                        mappingTransMeta = rep.loadTransformation(tmpFilename, dir, null, true, null);
                    } catch (KettleException ke) {
                        // try without extension
                        if (realFilename.endsWith(Const.STRING_TRANS_DEFAULT_EXT)) {
                            try {
                                String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1, realFilename.indexOf("." + Const.STRING_TRANS_DEFAULT_EXT));
                                String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
                                RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
                                mappingTransMeta = rep.loadTransformation(tmpFilename, dir, null, true, null);
                            } catch (KettleException ke2) {
                            // fall back to try loading from file system (transMeta is going to be null)
                            }
                        }
                    }
                }
                if (mappingTransMeta == null) {
                    mappingTransMeta = new TransMeta(realFilename, metaStore, rep, true, tmpSpace, null);
                    LogChannel.GENERAL.logDetailed("Loading transformation from repository", "Transformation was loaded from XML file [" + realFilename + "]");
                }
            } catch (Exception e) {
                throw new KettleException(BaseMessages.getString(PKG, "StepWithMappingMeta.Exception.UnableToLoadTrans"), e);
            }
            break;
        case REPOSITORY_BY_NAME:
            String realTransname = tmpSpace.environmentSubstitute(executorMeta.getTransName());
            String realDirectory = tmpSpace.environmentSubstitute(executorMeta.getDirectoryPath());
            if (rep != null) {
                if (!Utils.isEmpty(realTransname) && !Utils.isEmpty(realDirectory)) {
                    realDirectory = r.normalizeSlashes(realDirectory);
                    RepositoryDirectoryInterface repdir = rep.findDirectory(realDirectory);
                    if (repdir != null) {
                        try {
                            // reads the last revision in the repository...
                            mappingTransMeta = rep.loadTransformation(realTransname, repdir, null, true, null);
                            // TODO: FIXME: pass in metaStore to repository?
                            LogChannel.GENERAL.logDetailed("Loading transformation from repository", "Executor transformation [" + realTransname + "] was loaded from the repository");
                        } catch (Exception e) {
                            throw new KettleException("Unable to load transformation [" + realTransname + "]", e);
                        }
                    }
                }
            } else {
                // rep is null, let's try loading by filename
                try {
                    mappingTransMeta = new TransMeta(realDirectory + "/" + realTransname, metaStore, null, true, tmpSpace, null);
                } catch (KettleException ke) {
                    try {
                        // add .ktr extension and try again
                        mappingTransMeta = new TransMeta(realDirectory + "/" + realTransname + "." + Const.STRING_TRANS_DEFAULT_EXT, metaStore, null, true, tmpSpace, null);
                    } catch (KettleException ke2) {
                        throw new KettleException(BaseMessages.getString(PKG, "StepWithMappingMeta.Exception.UnableToLoadTrans", realTransname) + realDirectory);
                    }
                }
            }
            break;
        case REPOSITORY_BY_REFERENCE:
            // Read the last revision by reference...
            mappingTransMeta = rep.loadTransformation(executorMeta.getTransObjectId(), null);
            break;
        default:
            break;
    }
    if (mappingTransMeta == null) {
        // skip warning
        return null;
    }
    // When the child parameter does exist in the parent parameters, overwrite the child parameter by the
    // parent parameter.
    replaceVariableValues(mappingTransMeta, space);
    if (share) {
        // All other parent parameters need to get copied into the child parameters  (when the 'Inherit all
        // variables from the transformation?' option is checked)
        addMissingVariables(mappingTransMeta, space);
    }
    mappingTransMeta.setRepository(rep);
    mappingTransMeta.setMetaStore(metaStore);
    mappingTransMeta.setFilename(mappingTransMeta.getFilename());
    return mappingTransMeta;
}

Example 5 with KettleException

use of org.pentaho.di.core.exception.KettleException in project pentaho-kettle by pentaho.

the class GetXMLData method processPutRow.

private Object[] processPutRow(AbstractNode node) throws KettleException {
    // Create new row...
    Object[] outputRowData = buildEmptyRow();
    // Create new row or clone
    if (meta.isInFields()) {
        System.arraycopy(data.readrow, 0, outputRowData, 0, data.nrReadRow);
    }
    try {
        data.nodenr++;
        // Read fields...
        for (int i = 0; i < data.nrInputFields; i++) {
            // Get field
            GetXMLDataField xmlDataField = meta.getInputFields()[i];
            // Get the Path to look for
            String XPathValue = xmlDataField.getXPath();
            XPathValue = environmentSubstitute(XPathValue);
            if (xmlDataField.getElementType() == GetXMLDataField.ELEMENT_TYPE_ATTRIBUT) {
                // We have an attribute
                // do we need to add leading @?
                // Only put @ to the last element in path, not in front at all
                int last = XPathValue.lastIndexOf(GetXMLDataMeta.N0DE_SEPARATOR);
                if (last > -1) {
                    last++;
                    String attribut = XPathValue.substring(last, XPathValue.length());
                    if (!attribut.startsWith(GetXMLDataMeta.AT)) {
                        XPathValue = XPathValue.substring(0, last) + GetXMLDataMeta.AT + attribut;
                    }
                } else {
                    if (!XPathValue.startsWith(GetXMLDataMeta.AT)) {
                        XPathValue = GetXMLDataMeta.AT + XPathValue;
                    }
                }
            }
            if (meta.isuseToken()) {
                // See if user use Token inside path field
                // The syntax is : @_Fieldname-
                // PDI will search for Fieldname value and replace it
                // Fieldname must be defined before the current node
                XPathValue = substituteToken(XPathValue, outputRowData);
                if (isDetailed()) {
                    logDetailed(XPathValue);
                }
            }
            // Get node value
            String nodevalue;
            // Handle namespaces
            if (meta.isNamespaceAware()) {
                XPath xpathField = node.createXPath(addNSPrefix(XPathValue, data.PathValue));
                xpathField.setNamespaceURIs(data.NAMESPACE);
                if (xmlDataField.getResultType() == GetXMLDataField.RESULT_TYPE_VALUE_OF) {
                    nodevalue = xpathField.valueOf(node);
                } else {
                    // nodevalue=xpathField.selectSingleNode(node).asXML();
                    Node n = xpathField.selectSingleNode(node);
                    if (n != null) {
                        nodevalue = n.asXML();
                    } else {
                        nodevalue = "";
                    }
                }
            } else {
                if (xmlDataField.getResultType() == GetXMLDataField.RESULT_TYPE_VALUE_OF) {
                    nodevalue = node.valueOf(XPathValue);
                } else {
                    // nodevalue=node.selectSingleNode(XPathValue).asXML();
                    Node n = node.selectSingleNode(XPathValue);
                    if (n != null) {
                        nodevalue = n.asXML();
                    } else {
                        nodevalue = "";
                    }
                }
            }
            // Do trimming
            switch(xmlDataField.getTrimType()) {
                case GetXMLDataField.TYPE_TRIM_LEFT:
                    nodevalue = Const.ltrim(nodevalue);
                    break;
                case GetXMLDataField.TYPE_TRIM_RIGHT:
                    nodevalue = Const.rtrim(nodevalue);
                    break;
                case GetXMLDataField.TYPE_TRIM_BOTH:
                    nodevalue = Const.trim(nodevalue);
                    break;
                default:
                    break;
            }
            // Do conversions
            // 
            ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(data.totalpreviousfields + i);
            ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(data.totalpreviousfields + i);
            outputRowData[data.totalpreviousfields + i] = targetValueMeta.convertData(sourceValueMeta, nodevalue);
            // Do we need to repeat this field if it is null?
            if (meta.getInputFields()[i].isRepeated()) {
                if (data.previousRow != null && Utils.isEmpty(nodevalue)) {
                    outputRowData[data.totalpreviousfields + i] = data.previousRow[data.totalpreviousfields + i];
                }
            }
        }
        // End of loop over fields...
        int rowIndex = data.totalpreviousfields + data.nrInputFields;
        // See if we need to add the filename to the row...
        if (meta.includeFilename() && !Utils.isEmpty(meta.getFilenameField())) {
            outputRowData[rowIndex++] = data.filename;
        }
        // See if we need to add the row number to the row...
        if (meta.includeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) {
            outputRowData[rowIndex++] = data.rownr;
        }
        // Possibly add short filename...
        if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
            outputRowData[rowIndex++] = data.shortFilename;
        }
        // Add Extension
        if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
            outputRowData[rowIndex++] = data.extension;
        }
        // add path
        if (meta.getPathField() != null && meta.getPathField().length() > 0) {
            outputRowData[rowIndex++] = data.path;
        }
        // Add Size
        if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
            outputRowData[rowIndex++] = data.size;
        }
        // add Hidden
        if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
            outputRowData[rowIndex++] = Boolean.valueOf(data.path);
        }
        // Add modification date
        if (meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0) {
            outputRowData[rowIndex++] = data.lastModificationDateTime;
        }
        // Add Uri
        if (meta.getUriField() != null && meta.getUriField().length() > 0) {
            outputRowData[rowIndex++] = data.uriName;
        }
        // Add RootUri
        if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
            outputRowData[rowIndex] = data.rootUriName;
        }
        RowMetaInterface irow = getInputRowMeta();
        if (irow == null) {
            data.previousRow = outputRowData;
        } else {
            // clone to previously allocated array to make sure next step doesn't
            // change it in between...
            System.arraycopy(outputRowData, 0, this.prevRow, 0, outputRowData.length);
            // Pick up everything else that needs a real deep clone
            data.previousRow = irow.cloneRow(outputRowData, this.prevRow);
        }
    } catch (Exception e) {
        if (getStepMeta().isDoingErrorHandling()) {
            // Simply add this row to the error row
            putError(data.outputRowMeta, outputRowData, 1, e.toString(), null, "GetXMLData001");
            data.errorInRowButContinue = true;
            return null;
        } else {
            logError(e.toString());
            throw new KettleException(e.toString());
        }
    }
    return outputRowData;
}