Examples with CsvFileInfo org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo used on opensource projects

Example 1 with CsvFileInfo

use of org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo in project data-access by pentaho.

the class CsvPhysicalStep method refreshPreview.

@Bindable
public void refreshPreview() throws Exception {
    // $NON-NLS-1$
    csvTextPreview.setValue("");
    CsvFileInfo fileInfo = datasourceModel.getModelInfo().getFileInfo();
    csvTextPreview.setValue(fileInfo.formatSampleContents());
}

Example 2 with CsvFileInfo

use of org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo in project data-access by pentaho.

the class CsvDatasourceServiceImpl method prepareForSerialization.

protected void prepareForSerialization(Domain domain) throws IOException {
    /*
     * This method is responsible for cleaning up legacy information when
     * changing datasource types and also manages CSV files for CSV based
     * datasources.
     */
    String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
    String.valueOf(FileUtils.DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
    String path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    String TMP_FILE_PATH = File.separatorChar + "system" + File.separatorChar + File.separatorChar + "tmp" + File.separatorChar;
    String sysTmpDir = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    LogicalModel logicalModel = domain.getLogicalModels().get(0);
    // $NON-NLS-1$
    String modelState = (String) logicalModel.getProperty("datasourceModel");
    if (modelState != null) {
        XStream xs = new XStream();
        DatasourceDTO datasource = (DatasourceDTO) xs.fromXML(modelState);
        CsvFileInfo csvFileInfo = datasource.getCsvModelInfo().getFileInfo();
        String tmpFileName = csvFileInfo.getTmpFilename();
        String csvFileName = csvFileInfo.getFilename();
        File tmpFile = new File(sysTmpDir + File.separatorChar + tmpFileName);
        // Move CSV temporary file to final destination.
        if (tmpFile.exists()) {
            File csvFile = new File(path + File.separatorChar + csvFileName);
            org.apache.commons.io.FileUtils.copyFile(tmpFile, csvFile);
        }
        // Cleanup logic when updating from SQL datasource to CSV
        // datasource.
        datasource.setQuery(null);
        // Update datasourceModel with the new modelState
        modelState = xs.toXML(datasource);
        logicalModel.setProperty("datasourceModel", modelState);
    }
}

Example 3 with CsvFileInfo

use of org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo in project data-access by pentaho.

the class CsvTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    CsvFileInfo fileInfo = getModelInfo().getFileInfo();
    String fileName = fileInfo.getTmpFilename();
    String path;
    if (fileName.endsWith(".tmp")) {
        // $NON-NLS-1$
        path = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    } else {
        String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
        String.valueOf(DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
        path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    }
    File file = new File(path + fileInfo.getTmpFilename());
    String filename = file.getAbsolutePath();
    ColumnInfo[] columns = getModelInfo().getColumns();
    TextFileInputField[] inputFields = new TextFileInputField[columns.length];
    int idx = 0;
    for (ColumnInfo column : columns) {
        TextFileInputField field = new TextFileInputField();
        field.setCurrencySymbol(fileInfo.getCurrencySymbol());
        field.setDecimalSymbol(fileInfo.getCurrencySymbol());
        field.setFormat(column.getFormat());
        field.setGroupSymbol(fileInfo.getGroupSymbol());
        field.setIfNullValue(fileInfo.getIfNull());
        field.setIgnored(column.isIgnore());
        field.setLength(column.getLength());
        field.setName(column.getId());
        field.setNullString(fileInfo.getNullStr());
        // field.setPosition(position);
        field.setPrecision(column.getPrecision());
        field.setRepeated(false);
        field.setSamples(null);
        field.setTrimType(ValueMeta.TRIM_TYPE_BOTH);
        field.setType(convertDataType(column));
        inputFields[idx] = field;
        idx++;
    }
    csvInputMeta.setAddResultFile(false);
    // $NON-NLS-1$
    csvInputMeta.setBufferSize("5000");
    csvInputMeta.setDelimiter(fileInfo.getDelimiter());
    csvInputMeta.setEnclosure(fileInfo.getEnclosure());
    csvInputMeta.setEncoding(fileInfo.getEncoding());
    csvInputMeta.setFilename(filename);
    csvInputMeta.setFilenameField(null);
    // TODO strip off more than one row if present...
    csvInputMeta.setHeaderPresent(fileInfo.getHeaderRows() > 0);
    // inputMeta.get.setID(1);
    csvInputMeta.setIncludingFilename(false);
    csvInputMeta.setInputFields(inputFields);
    csvInputMeta.setLazyConversionActive(true);
    // $NON-NLS-1$
    csvInputMeta.setRowNumField("");
    csvInputMeta.setRunningInParallel(false);
    // inputMeta.setTargetSteps(null);
    StepMeta csvInputStepMeta = new StepMeta(CSV_INPUT, CSV_INPUT, csvInputMeta);
    csvInputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, csvInputStepMeta));
    transMeta.addStep(csvInputStepMeta);
    csvErrorRowCount = 0;
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta csvInputErrorMeta = new StepErrorMeta(transMeta, csvInputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            if (csvErrorRowCount < maxErrorRows) {
                StringBuffer sb = new StringBuffer();
                sb.append("Rejected Row: ");
                for (Object rowData : row) {
                    sb.append(rowData);
                    sb.append(", ");
                }
                sb.append("\r\n");
                stats.getErrors().add(sb.toString() + errorDescriptions);
            }
            csvErrorRowCount++;
            stats.setErrorCount(csvErrorRowCount);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "CSVInputErrorDummy");
    csvInputErrorMeta.setTargetStep(outputDummyStepMeta);
    csvInputErrorMeta.setEnabled(true);
    csvInputStepMeta.setStepErrorMeta(csvInputErrorMeta);
    return csvInputStepMeta;
}

Example 4 with CsvFileInfo

use of org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo in project data-access by pentaho.

the class CsvDatasourceServiceImplTest method testHasPermissions.

@Test
public void testHasPermissions() throws Exception {
    hasPermissions = true;
    final ISystemSettings systemSettings = mock(ISystemSettings.class);
    when(systemSettings.getSystemSetting("data-access-override", "false")).thenReturn("false");
    PentahoSystem.setSystemSettingsService(systemSettings);
    String filename = "anotherStageFile_CsvFile.csv";
    File file = createTmpCsvFile(filename);
    file.deleteOnExit();
    try {
        ModelInfo modelInfo = service.stageFile(filename, ",", "\n", true, "utf-8");
        CsvFileInfo fileInfo = modelInfo.getFileInfo();
        assertEquals("One header row", 1, fileInfo.getHeaderRows());
        assertEquals("Header + content row", 2, fileInfo.getContents().size());
        assertEquals(filename, fileInfo.getTmpFilename());
        final FileInfo[] stagedFiles = service.getStagedFiles();
        assertNotNull(stagedFiles);
        boolean present = false;
        for (FileInfo info : stagedFiles) {
            if (filename.equals(info.getName())) {
                present = true;
                break;
            }
        }
        assertTrue(present);
        final String encoding = service.getEncoding(filename);
        assertNotNull(encoding);
        final List<String> previewRows = service.getPreviewRows(filename, true, 1, "utf-8");
        assertNotNull(previewRows);
        assertEquals(1, previewRows.size());
        assertEquals("col1,col2", previewRows.get(0));
        final DatasourceDTO datasourceDto = mock(DatasourceDTO.class);
        when(datasourceDto.getCsvModelInfo()).thenReturn(modelInfo);
        try {
            final FileTransformStats fileTransformStats = service.generateDomain(datasourceDto);
        } catch (Exception e) {
        // Testing this logic is not a purpose of this junit
        }
        // Passed permissions check
        verify(datasourceDto, times(1)).getCsvModelInfo();
    } finally {
        file.delete();
    }
}

Example 5 with CsvFileInfo

use of org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo in project data-access by pentaho.

the class CsvDatasourceServiceImplTest method stageFile_CsvFile.

@Test
public void stageFile_CsvFile() throws Exception {
    String filename = "stageFile_CsvFile.csv";
    File file = createTmpCsvFile(filename);
    file.deleteOnExit();
    try {
        ModelInfo modelInfo = service.stageFile(filename, ",", "\n", true, "utf-8");
        CsvFileInfo fileInfo = modelInfo.getFileInfo();
        assertEquals("One header row", 1, fileInfo.getHeaderRows());
        assertEquals("Header + content row", 2, fileInfo.getContents().size());
        assertEquals(filename, fileInfo.getTmpFilename());
    } finally {
        file.delete();
    }
}