Examples with FileTransformStats org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats used on opensource projects

Example 1 with FileTransformStats

use of org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats in project data-access by pentaho.

the class CsvTransformGeneratorIT method loadTable.

private int loadTable(CsvTransformGenerator gen, ModelInfo info, boolean truncate, IPentahoSession session) throws InterruptedException, CsvTransformGeneratorException {
    gen.loadTable(truncate, session, false);
    FileTransformStats stats = gen.getTransformStats();
    // wait until it it done
    while (!stats.isRowsFinished()) {
        Thread.sleep(100);
    }
    return 1;
}

Example 2 with FileTransformStats

use of org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats in project data-access by pentaho.

the class TableInputTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    TableInputMeta inputMeta = new TableInputMeta();
    inputMeta.setDatabaseMeta(sourceDatabaseMeta);
    inputMeta.setExecuteEachInputRow(false);
    inputMeta.setRowLimit(Integer.toString(rowLimit));
    inputMeta.setSQL(sql);
    inputMeta.setVariableReplacementActive(false);
    inputMeta.setLazyConversionActive(false);
    // inputMeta.setTargetSteps(null);
    StepMeta inputStepMeta = new StepMeta(TABLE_INPUT, TABLE_INPUT, inputMeta);
    inputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, inputStepMeta));
    transMeta.addStep(inputStepMeta);
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta inputErrorMeta = new StepErrorMeta(transMeta, inputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            StringBuffer sb = new StringBuffer();
            sb.append("Rejected Row: ");
            for (Object rowData : row) {
                sb.append(rowData);
                sb.append(", ");
            }
            sb.append("\r\n");
            stats.getErrors().add(sb.toString() + errorDescriptions);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "InputErrorDummy");
    inputErrorMeta.setTargetStep(outputDummyStepMeta);
    inputErrorMeta.setEnabled(true);
    inputStepMeta.setStepErrorMeta(inputErrorMeta);
    return inputStepMeta;
}

Example 3 with FileTransformStats

use of org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats in project data-access by pentaho.

the class CsvTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    CsvFileInfo fileInfo = getModelInfo().getFileInfo();
    String fileName = fileInfo.getTmpFilename();
    String path;
    if (fileName.endsWith(".tmp")) {
        // $NON-NLS-1$
        path = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    } else {
        String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
        String.valueOf(DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
        path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    }
    File file = new File(path + fileInfo.getTmpFilename());
    String filename = file.getAbsolutePath();
    ColumnInfo[] columns = getModelInfo().getColumns();
    TextFileInputField[] inputFields = new TextFileInputField[columns.length];
    int idx = 0;
    for (ColumnInfo column : columns) {
        TextFileInputField field = new TextFileInputField();
        field.setCurrencySymbol(fileInfo.getCurrencySymbol());
        field.setDecimalSymbol(fileInfo.getCurrencySymbol());
        field.setFormat(column.getFormat());
        field.setGroupSymbol(fileInfo.getGroupSymbol());
        field.setIfNullValue(fileInfo.getIfNull());
        field.setIgnored(column.isIgnore());
        field.setLength(column.getLength());
        field.setName(column.getId());
        field.setNullString(fileInfo.getNullStr());
        // field.setPosition(position);
        field.setPrecision(column.getPrecision());
        field.setRepeated(false);
        field.setSamples(null);
        field.setTrimType(ValueMeta.TRIM_TYPE_BOTH);
        field.setType(convertDataType(column));
        inputFields[idx] = field;
        idx++;
    }
    csvInputMeta.setAddResultFile(false);
    // $NON-NLS-1$
    csvInputMeta.setBufferSize("5000");
    csvInputMeta.setDelimiter(fileInfo.getDelimiter());
    csvInputMeta.setEnclosure(fileInfo.getEnclosure());
    csvInputMeta.setEncoding(fileInfo.getEncoding());
    csvInputMeta.setFilename(filename);
    csvInputMeta.setFilenameField(null);
    // TODO strip off more than one row if present...
    csvInputMeta.setHeaderPresent(fileInfo.getHeaderRows() > 0);
    // inputMeta.get.setID(1);
    csvInputMeta.setIncludingFilename(false);
    csvInputMeta.setInputFields(inputFields);
    csvInputMeta.setLazyConversionActive(true);
    // $NON-NLS-1$
    csvInputMeta.setRowNumField("");
    csvInputMeta.setRunningInParallel(false);
    // inputMeta.setTargetSteps(null);
    StepMeta csvInputStepMeta = new StepMeta(CSV_INPUT, CSV_INPUT, csvInputMeta);
    csvInputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, csvInputStepMeta));
    transMeta.addStep(csvInputStepMeta);
    csvErrorRowCount = 0;
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta csvInputErrorMeta = new StepErrorMeta(transMeta, csvInputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            if (csvErrorRowCount < maxErrorRows) {
                StringBuffer sb = new StringBuffer();
                sb.append("Rejected Row: ");
                for (Object rowData : row) {
                    sb.append(rowData);
                    sb.append(", ");
                }
                sb.append("\r\n");
                stats.getErrors().add(sb.toString() + errorDescriptions);
            }
            csvErrorRowCount++;
            stats.setErrorCount(csvErrorRowCount);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "CSVInputErrorDummy");
    csvInputErrorMeta.setTargetStep(outputDummyStepMeta);
    csvInputErrorMeta.setEnabled(true);
    csvInputStepMeta.setStepErrorMeta(csvInputErrorMeta);
    return csvInputStepMeta;
}

Example 4 with FileTransformStats

use of org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats in project data-access by pentaho.

the class CsvDatasourceServiceImplTest method testHasPermissions.

@Test
public void testHasPermissions() throws Exception {
    hasPermissions = true;
    final ISystemSettings systemSettings = mock(ISystemSettings.class);
    when(systemSettings.getSystemSetting("data-access-override", "false")).thenReturn("false");
    PentahoSystem.setSystemSettingsService(systemSettings);
    String filename = "anotherStageFile_CsvFile.csv";
    File file = createTmpCsvFile(filename);
    file.deleteOnExit();
    try {
        ModelInfo modelInfo = service.stageFile(filename, ",", "\n", true, "utf-8");
        CsvFileInfo fileInfo = modelInfo.getFileInfo();
        assertEquals("One header row", 1, fileInfo.getHeaderRows());
        assertEquals("Header + content row", 2, fileInfo.getContents().size());
        assertEquals(filename, fileInfo.getTmpFilename());
        final FileInfo[] stagedFiles = service.getStagedFiles();
        assertNotNull(stagedFiles);
        boolean present = false;
        for (FileInfo info : stagedFiles) {
            if (filename.equals(info.getName())) {
                present = true;
                break;
            }
        }
        assertTrue(present);
        final String encoding = service.getEncoding(filename);
        assertNotNull(encoding);
        final List<String> previewRows = service.getPreviewRows(filename, true, 1, "utf-8");
        assertNotNull(previewRows);
        assertEquals(1, previewRows.size());
        assertEquals("col1,col2", previewRows.get(0));
        final DatasourceDTO datasourceDto = mock(DatasourceDTO.class);
        when(datasourceDto.getCsvModelInfo()).thenReturn(modelInfo);
        try {
            final FileTransformStats fileTransformStats = service.generateDomain(datasourceDto);
        } catch (Exception e) {
        // Testing this logic is not a purpose of this junit
        }
        // Passed permissions check
        verify(datasourceDto, times(1)).getCsvModelInfo();
    } finally {
        file.delete();
    }
}

Example 5 with FileTransformStats

use of org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats in project data-access by pentaho.

the class CsvDatasourceServiceImpl method generateDomain.

public FileTransformStats generateDomain(DatasourceDTO datasourceDto) throws Exception {
    checkPermissions();
    synchronized (lock) {
        ModelInfo modelInfo = datasourceDto.getCsvModelInfo();
        IPentahoSession pentahoSession = null;
        try {
            pentahoSession = PentahoSessionHolder.getSession();
            KettleSystemListener.environmentInit(pentahoSession);
            String statsKey = // $NON-NLS-1$
            FileTransformStats.class.getSimpleName() + "_" + modelInfo.getFileInfo().getTmpFilename();
            FileTransformStats stats = new FileTransformStats();
            pentahoSession.setAttribute(statsKey, stats);
            CsvTransformGenerator csvTransformGenerator = new CsvTransformGenerator(modelInfo, AgileHelper.getDatabaseMeta());
            csvTransformGenerator.setTransformStats(stats);
            try {
                csvTransformGenerator.dropTable(modelInfo.getStageTableName());
            } catch (CsvTransformGeneratorException e) {
                // this is ok, the table may not have existed.
                // $NON-NLS-1$
                logger.info("Could not drop table before staging");
            }
            csvTransformGenerator.createOrModifyTable(pentahoSession);
            // no longer need to truncate the table since we dropped it a few lines up, so just pass false
            csvTransformGenerator.loadTable(false, pentahoSession, true);
            ArrayList<String> combinedErrors = new ArrayList<String>(modelInfo.getCsvInputErrors());
            combinedErrors.addAll(modelInfo.getTableOutputErrors());
            if (stats.getErrors() != null && stats.getErrors().size() > 0) {
                stats.getErrors().addAll(combinedErrors);
            } else {
                stats.setErrors(combinedErrors);
            }
            // wait until it it done
            while (!stats.isRowsFinished()) {
                Thread.sleep(200);
            }
            modelerWorkspace.setDomain(modelerService.generateCSVDomain(modelInfo));
            modelerWorkspace.getWorkspaceHelper().autoModelFlat(modelerWorkspace);
            modelerWorkspace.getWorkspaceHelper().autoModelRelationalFlat(modelerWorkspace);
            modelerWorkspace.setModelName(modelInfo.getDatasourceName());
            modelerWorkspace.getWorkspaceHelper().populateDomain(modelerWorkspace);
            Domain workspaceDomain = modelerWorkspace.getDomain();
            XStream xstream = new XStream();
            String serializedDto = xstream.toXML(datasourceDto);
            workspaceDomain.getLogicalModels().get(0).setProperty("datasourceModel", serializedDto);
            workspaceDomain.getLogicalModels().get(0).setProperty("DatasourceType", "CSV");
            prepareForSerialization(workspaceDomain);
            modelerService.serializeModels(workspaceDomain, modelerWorkspace.getModelName());
            stats.setDomain(modelerWorkspace.getDomain());
            return stats;
        } catch (Exception e) {
            logger.error(e.getMessage());
            throw e;
        } finally {
            if (pentahoSession != null) {
                pentahoSession.destroy();
            }
        }
    }
}