Examples with CsvInputMeta org.pentaho.di.trans.steps.csvinput.CsvInputMeta used on opensource projects

Example 1 with CsvInputMeta

use of org.pentaho.di.trans.steps.csvinput.CsvInputMeta in project data-access by pentaho.

the class CsvTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    CsvFileInfo fileInfo = getModelInfo().getFileInfo();
    String fileName = fileInfo.getTmpFilename();
    String path;
    if (fileName.endsWith(".tmp")) {
        // $NON-NLS-1$
        path = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    } else {
        String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
        String.valueOf(DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
        path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    }
    File file = new File(path + fileInfo.getTmpFilename());
    String filename = file.getAbsolutePath();
    ColumnInfo[] columns = getModelInfo().getColumns();
    TextFileInputField[] inputFields = new TextFileInputField[columns.length];
    int idx = 0;
    for (ColumnInfo column : columns) {
        TextFileInputField field = new TextFileInputField();
        field.setCurrencySymbol(fileInfo.getCurrencySymbol());
        field.setDecimalSymbol(fileInfo.getCurrencySymbol());
        field.setFormat(column.getFormat());
        field.setGroupSymbol(fileInfo.getGroupSymbol());
        field.setIfNullValue(fileInfo.getIfNull());
        field.setIgnored(column.isIgnore());
        field.setLength(column.getLength());
        field.setName(column.getId());
        field.setNullString(fileInfo.getNullStr());
        // field.setPosition(position);
        field.setPrecision(column.getPrecision());
        field.setRepeated(false);
        field.setSamples(null);
        field.setTrimType(ValueMeta.TRIM_TYPE_BOTH);
        field.setType(convertDataType(column));
        inputFields[idx] = field;
        idx++;
    }
    csvInputMeta.setAddResultFile(false);
    // $NON-NLS-1$
    csvInputMeta.setBufferSize("5000");
    csvInputMeta.setDelimiter(fileInfo.getDelimiter());
    csvInputMeta.setEnclosure(fileInfo.getEnclosure());
    csvInputMeta.setEncoding(fileInfo.getEncoding());
    csvInputMeta.setFilename(filename);
    csvInputMeta.setFilenameField(null);
    // TODO strip off more than one row if present...
    csvInputMeta.setHeaderPresent(fileInfo.getHeaderRows() > 0);
    // inputMeta.get.setID(1);
    csvInputMeta.setIncludingFilename(false);
    csvInputMeta.setInputFields(inputFields);
    csvInputMeta.setLazyConversionActive(true);
    // $NON-NLS-1$
    csvInputMeta.setRowNumField("");
    csvInputMeta.setRunningInParallel(false);
    // inputMeta.setTargetSteps(null);
    StepMeta csvInputStepMeta = new StepMeta(CSV_INPUT, CSV_INPUT, csvInputMeta);
    csvInputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, csvInputStepMeta));
    transMeta.addStep(csvInputStepMeta);
    csvErrorRowCount = 0;
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta csvInputErrorMeta = new StepErrorMeta(transMeta, csvInputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            if (csvErrorRowCount < maxErrorRows) {
                StringBuffer sb = new StringBuffer();
                sb.append("Rejected Row: ");
                for (Object rowData : row) {
                    sb.append(rowData);
                    sb.append(", ");
                }
                sb.append("\r\n");
                stats.getErrors().add(sb.toString() + errorDescriptions);
            }
            csvErrorRowCount++;
            stats.setErrorCount(csvErrorRowCount);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "CSVInputErrorDummy");
    csvInputErrorMeta.setTargetStep(outputDummyStepMeta);
    csvInputErrorMeta.setEnabled(true);
    csvInputStepMeta.setStepErrorMeta(csvInputErrorMeta);
    return csvInputStepMeta;
}

Example 2 with CsvInputMeta

use of org.pentaho.di.trans.steps.csvinput.CsvInputMeta in project pentaho-kettle by pentaho.

the class TransMetaConverterTest method lazyConversionTurnedOff.

@Test
public void lazyConversionTurnedOff() throws KettleException {
    KettleEnvironment.init();
    TransMeta transMeta = new TransMeta();
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    csvInputMeta.setLazyConversionActive(true);
    StepMeta csvInput = new StepMeta("Csv", csvInputMeta);
    transMeta.addStep(csvInput);
    TableInputMeta tableInputMeta = new TableInputMeta();
    tableInputMeta.setLazyConversionActive(true);
    StepMeta tableInput = new StepMeta("Table", tableInputMeta);
    transMeta.addStep(tableInput);
    Transformation trans = TransMetaConverter.convert(transMeta);
    TransMeta cloneMeta;
    String transMetaXml = (String) trans.getConfig().get(TransMetaConverter.TRANS_META_CONF_KEY);
    Document doc;
    try {
        doc = XMLHandler.loadXMLString(transMetaXml);
        Node stepNode = XMLHandler.getSubNode(doc, "transformation");
        cloneMeta = new TransMeta(stepNode, null);
    } catch (KettleXMLException | KettleMissingPluginsException e) {
        throw new RuntimeException(e);
    }
    assertThat(((CsvInputMeta) cloneMeta.findStep("Csv").getStepMetaInterface()).isLazyConversionActive(), is(false));
    assertThat(((TableInputMeta) cloneMeta.findStep("Table").getStepMetaInterface()).isLazyConversionActive(), is(false));
}

Example 3 with CsvInputMeta

use of org.pentaho.di.trans.steps.csvinput.CsvInputMeta in project pentaho-kettle by pentaho.

the class SpoonTest method testCopyPasteOneStepWithErrorHandling.

/**
 * test copy one step with error handling
 * @see http://jira.pentaho.com/browse/PDI-13358
 *
 * @throws KettleException
 */
@Test
public void testCopyPasteOneStepWithErrorHandling() throws KettleException {
    final TransMeta transMeta = new TransMeta();
    StepMeta sourceStep = new StepMeta("CsvInput", "Step1", new CsvInputMeta());
    StepMeta targetStep = new StepMeta("Dummy", "Dummy Step1", new DummyTransMeta());
    sourceStep.setSelected(true);
    transMeta.addStep(sourceStep);
    transMeta.addStep(targetStep);
    StepErrorMeta errorMeta = new StepErrorMeta(transMeta, sourceStep, targetStep);
    sourceStep.setStepErrorMeta(errorMeta);
    errorMeta.setSourceStep(sourceStep);
    errorMeta.setTargetStep(targetStep);
    final int stepsSizeBefore = transMeta.getSteps().size();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            spoon.pasteXML(transMeta, (String) invocation.getArguments()[0], mock(Point.class));
            assertTrue("Steps was not copied", stepsSizeBefore < transMeta.getSteps().size());
            // selected copied step
            for (StepMeta s : transMeta.getSelectedSteps()) {
                if (s.getStepMetaInterface() instanceof CsvInputMeta) {
                    // check that stepError was empty, because we copy only one step from pair
                    assertNull("Error hop was not copied", s.getStepErrorMeta());
                }
            }
            return null;
        }
    }).when(spoon).toClipboard(anyString());
    spoon.copySelected(transMeta, transMeta.getSelectedSteps(), Collections.<NotePadMeta>emptyList());
}

Example 4 with CsvInputMeta

use of org.pentaho.di.trans.steps.csvinput.CsvInputMeta in project pentaho-kettle by pentaho.

the class SpoonTest method testCopyPasteStepsErrorHandling.

/**
 * test two steps
 * @see http://jira.pentaho.com/browse/PDI-689
 *
 * @throws KettleException
 */
@Test
public void testCopyPasteStepsErrorHandling() throws KettleException {
    final TransMeta transMeta = new TransMeta();
    // for check copy both step and hop
    StepMeta sourceStep = new StepMeta("CsvInput", "Step1", new CsvInputMeta());
    StepMeta targetStep = new StepMeta("Dummy", "Dummy Step1", new DummyTransMeta());
    sourceStep.setSelected(true);
    targetStep.setSelected(true);
    transMeta.addStep(sourceStep);
    transMeta.addStep(targetStep);
    StepErrorMeta errorMeta = new StepErrorMeta(transMeta, sourceStep, targetStep);
    sourceStep.setStepErrorMeta(errorMeta);
    errorMeta.setSourceStep(sourceStep);
    errorMeta.setTargetStep(targetStep);
    final int stepsSizeBefore = transMeta.getSteps().size();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            spoon.pasteXML(transMeta, (String) invocation.getArguments()[0], mock(Point.class));
            assertTrue("Steps was not copied", stepsSizeBefore < transMeta.getSteps().size());
            // selected copied step
            for (StepMeta s : transMeta.getSelectedSteps()) {
                if (s.getStepMetaInterface() instanceof CsvInputMeta) {
                    // check that stepError was copied
                    assertNotNull("Error hop was not copied", s.getStepErrorMeta());
                }
            }
            return null;
        }
    }).when(spoon).toClipboard(anyString());
    spoon.copySelected(transMeta, transMeta.getSelectedSteps(), Collections.<NotePadMeta>emptyList());
}

Example 5 with CsvInputMeta

use of org.pentaho.di.trans.steps.csvinput.CsvInputMeta in project pentaho-kettle by pentaho.

the class CsvInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    InputStream inputStream = null;
    try {
        CsvInputMeta meta = new CsvInputMeta();
        getInfo(meta);
        String filename = transMeta.environmentSubstitute(meta.getFilename());
        String delimiter = transMeta.environmentSubstitute(meta.getDelimiter());
        String enclosure = transMeta.environmentSubstitute(meta.getEnclosure());
        FileObject fileObject = KettleVFS.getFileObject(filename);
        if (!(fileObject instanceof LocalFile)) {
            // 
            throw new KettleException(BaseMessages.getString(PKG, "CsvInput.Log.OnlyLocalFilesAreSupported"));
        }
        wFields.table.removeAll();
        inputStream = KettleVFS.getInputStream(fileObject);
        String realEncoding = transMeta.environmentSubstitute(meta.getEncoding());
        InputStreamReader reader;
        if (Utils.isEmpty(realEncoding)) {
            reader = new InputStreamReader(inputStream);
        } else {
            reader = new InputStreamReader(inputStream, realEncoding);
        }
        EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
        // Read a line of data to determine the number of rows...
        // 
        String line = TextFileInput.getLine(log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder(1000));
        // Split the string, header or data into parts...
        // 
        String[] fieldNames = CsvInput.guessStringsFromLine(log, line, delimiter, enclosure, meta.getEscapeCharacter());
        if (!meta.isHeaderPresent()) {
            // Don't use field names from the header...
            // Generate field names F1 ... F10
            // 
            DecimalFormat df = new DecimalFormat("000");
            for (int i = 0; i < fieldNames.length; i++) {
                fieldNames[i] = "Field_" + df.format(i);
            }
        } else {
            if (!Utils.isEmpty(meta.getEnclosure())) {
                for (int i = 0; i < fieldNames.length; i++) {
                    if (fieldNames[i].startsWith(meta.getEnclosure()) && fieldNames[i].endsWith(meta.getEnclosure()) && fieldNames[i].length() > 1) {
                        fieldNames[i] = fieldNames[i].substring(1, fieldNames[i].length() - 1);
                    }
                }
            }
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            fieldNames[i] = Const.trim(fieldNames[i]);
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            TableItem item = new TableItem(wFields.table, SWT.NONE);
            item.setText(1, fieldNames[i]);
            item.setText(2, ValueMetaFactory.getValueMetaName(ValueMetaInterface.TYPE_STRING));
        }
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth(true);
        // Now we can continue reading the rows of data and we can guess the
        // Sample a few lines to determine the correct type of the fields...
        // 
        String shellText = BaseMessages.getString(PKG, "CsvInputDialog.LinesToSample.DialogTitle");
        String lineText = BaseMessages.getString(PKG, "CsvInputDialog.LinesToSample.DialogMessage");
        EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
        int samples = end.open();
        if (samples >= 0) {
            getInfo(meta);
            TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true);
            String message = pd.open();
            if (message != null) {
                wFields.removeAll();
                // OK, what's the result of our search?
                getData(meta, false);
                wFields.removeEmptyRows();
                wFields.setRowNums();
                wFields.optWidth(true);
                EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "CsvInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "CsvInputDialog.ScanResults.DialogMessage"), message, true);
                etd.setReadOnly();
                etd.open();
            // asyncUpdatePreview();
            }
        }
    } catch (IOException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "CsvInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "CsvInputDialog.IOError.DialogMessage"), e);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "CsvInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
    } finally {
        try {
            inputStream.close();
        } catch (Exception e) {
        // Ignore close errors
        }
    }
}