Examples with BufferedOutputStreamWithCloseDetection org.pentaho.di.workarounds.BufferedOutputStreamWithCloseDetection used on opensource projects

Example 1 with BufferedOutputStreamWithCloseDetection

use of org.pentaho.di.workarounds.BufferedOutputStreamWithCloseDetection in project pentaho-kettle by pentaho.

the class JobEntryZipFile method processRowFile.

public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realSourceDirectoryOrFile, String realMovetodirectory, boolean createparentfolder) {
    boolean Fileexists = false;
    File tempFile = null;
    File fileZip;
    boolean resultat = false;
    boolean renameOk = false;
    boolean orginExist = false;
    // Check if target file/folder exists!
    FileObject originFile = null;
    ZipInputStream zin = null;
    byte[] buffer;
    OutputStream dest = null;
    BufferedOutputStreamWithCloseDetection buff = null;
    ZipOutputStream out = null;
    ZipEntry entry;
    String localSourceFilename = realSourceDirectoryOrFile;
    try {
        originFile = KettleVFS.getFileObject(realSourceDirectoryOrFile, this);
        localSourceFilename = KettleVFS.getFilename(originFile);
        orginExist = originFile.exists();
    } catch (Exception e) {
    // Ignore errors
    } finally {
        if (originFile != null) {
            try {
                originFile.close();
            } catch (IOException ex) {
                logError("Error closing file '" + originFile.toString() + "'", ex);
            }
        }
    }
    String localrealZipfilename = realZipfilename;
    if (realZipfilename != null && orginExist) {
        FileObject fileObject = null;
        try {
            fileObject = KettleVFS.getFileObject(localrealZipfilename, this);
            localrealZipfilename = KettleVFS.getFilename(fileObject);
            // Check if Zip File exists
            if (fileObject.exists()) {
                Fileexists = true;
                if (log.isDebug()) {
                    logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists2.Label"));
                }
            }
            // Let's see if we need to create parent folder of destination zip filename
            if (createparentfolder) {
                createParentFolder(localrealZipfilename);
            }
            // Let's start the process now
            if (ifZipFileExists == 3 && Fileexists) {
                // the zip file exists and user want to Fail
                resultat = false;
            } else if (ifZipFileExists == 2 && Fileexists) {
                // the zip file exists and user want to do nothing
                if (addFileToResult) {
                    // Add file to result files name
                    ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
                    result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
                }
                resultat = true;
            } else if (afterZip == 2 && realMovetodirectory == null) {
                // After Zip, Move files..User must give a destination Folder
                resultat = false;
                logError(BaseMessages.getString(PKG, "JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label"));
            } else {
                // After Zip, Move files..User must give a destination Folder
                // Let's see if we deal with file or folder
                FileObject[] fileList;
                FileObject sourceFileOrFolder = KettleVFS.getFileObject(localSourceFilename, this);
                boolean isSourceDirectory = sourceFileOrFolder.getType().equals(FileType.FOLDER);
                final Pattern pattern;
                final Pattern patternExclude;
                if (isSourceDirectory) {
                    // 
                    if (!Utils.isEmpty(realWildcard)) {
                        pattern = Pattern.compile(realWildcard);
                    } else {
                        pattern = null;
                    }
                    if (!Utils.isEmpty(realWildcardExclude)) {
                        patternExclude = Pattern.compile(realWildcardExclude);
                    } else {
                        patternExclude = null;
                    }
                    // 
                    if (includingSubFolders) {
                        fileList = sourceFileOrFolder.findFiles(new ZipJobEntryPatternFileSelector(pattern, patternExclude));
                    } else {
                        fileList = sourceFileOrFolder.getChildren();
                    }
                } else {
                    pattern = null;
                    patternExclude = null;
                    // Target is a file
                    fileList = new FileObject[] { sourceFileOrFolder };
                }
                if (fileList.length == 0) {
                    resultat = false;
                    logError(BaseMessages.getString(PKG, "JobZipFiles.Log.FolderIsEmpty", localSourceFilename));
                } else if (!checkContainsFile(localSourceFilename, fileList, isSourceDirectory)) {
                    resultat = false;
                    logError(BaseMessages.getString(PKG, "JobZipFiles.Log.NoFilesInFolder", localSourceFilename));
                } else {
                    if (ifZipFileExists == 0 && Fileexists) {
                        // do we have already a .zip at the end?
                        if (localrealZipfilename.toLowerCase().endsWith(".zip")) {
                            // strip this off
                            localrealZipfilename = localrealZipfilename.substring(0, localrealZipfilename.length() - 4);
                        }
                        localrealZipfilename += "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip";
                        if (log.isDebug()) {
                            logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label"));
                        }
                    } else if (ifZipFileExists == 1 && Fileexists) {
                        // the zip file exists and user want to append
                        // get a temp file
                        fileZip = getFile(localrealZipfilename);
                        tempFile = File.createTempFile(fileZip.getName(), null);
                        // delete it, otherwise we cannot rename existing zip to it.
                        tempFile.delete();
                        renameOk = fileZip.renameTo(tempFile);
                        if (!renameOk) {
                            logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp3.Label"));
                        }
                        if (log.isDebug()) {
                            logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend2.Label"));
                        }
                    }
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "JobZipFiles.Files_Found1.Label") + fileList.length + BaseMessages.getString(PKG, "JobZipFiles.Files_Found2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Files_Found3.Label"));
                    }
                    // Prepare Zip File
                    buffer = new byte[18024];
                    dest = KettleVFS.getOutputStream(localrealZipfilename, this, false);
                    buff = new BufferedOutputStreamWithCloseDetection(dest);
                    out = new ZipOutputStream(buff);
                    HashSet<String> fileSet = new HashSet<String>();
                    if (renameOk) {
                        // User want to append files to existing Zip file
                        // The idea is to rename the existing zip file to a temporary file
                        // and then adds all entries in the existing zip along with the new files,
                        // excluding the zip entries that have the same name as one of the new files.
                        zin = new ZipInputStream(new FileInputStream(tempFile));
                        entry = zin.getNextEntry();
                        while (entry != null) {
                            String name = entry.getName();
                            if (!fileSet.contains(name)) {
                                // Add ZIP entry to output stream.
                                out.putNextEntry(new ZipEntry(name));
                                // Transfer bytes from the ZIP file to the output file
                                int len;
                                while ((len = zin.read(buffer)) > 0) {
                                    out.write(buffer, 0, len);
                                }
                                fileSet.add(name);
                            }
                            entry = zin.getNextEntry();
                        }
                        // Close the streams
                        zin.close();
                    }
                    // Set the method
                    out.setMethod(ZipOutputStream.DEFLATED);
                    // Set the compression level
                    if (compressionRate == 0) {
                        out.setLevel(Deflater.NO_COMPRESSION);
                    } else if (compressionRate == 1) {
                        out.setLevel(Deflater.DEFAULT_COMPRESSION);
                    }
                    if (compressionRate == 2) {
                        out.setLevel(Deflater.BEST_COMPRESSION);
                    }
                    if (compressionRate == 3) {
                        out.setLevel(Deflater.BEST_SPEED);
                    }
                    // Specify Zipped files (After that we will move,delete them...)
                    FileObject[] zippedFiles = new FileObject[fileList.length];
                    int fileNum = 0;
                    // Get the files in the list...
                    for (int i = 0; i < fileList.length && !parentJob.isStopped(); i++) {
                        boolean getIt = true;
                        boolean getItexclude = false;
                        // ..only if target is a folder !
                        if (isSourceDirectory) {
                            // If we include sub-folders, we match on the whole name, not just the basename
                            // 
                            String filename;
                            if (includingSubFolders) {
                                filename = fileList[i].getName().getPath();
                            } else {
                                filename = fileList[i].getName().getBaseName();
                            }
                            if (pattern != null) {
                                // Matches the base name of the file (backward compatible!)
                                // 
                                Matcher matcher = pattern.matcher(filename);
                                getIt = matcher.matches();
                            }
                            if (patternExclude != null) {
                                Matcher matcherexclude = patternExclude.matcher(filename);
                                getItexclude = matcherexclude.matches();
                            }
                        }
                        // Get processing File
                        String targetFilename = KettleVFS.getFilename(fileList[i]);
                        if (sourceFileOrFolder.getType().equals(FileType.FILE)) {
                            targetFilename = localSourceFilename;
                        }
                        FileObject file = KettleVFS.getFileObject(targetFilename, this);
                        boolean isTargetDirectory = file.exists() && file.getType().equals(FileType.FOLDER);
                        if (getIt && !getItexclude && !isTargetDirectory && !fileSet.contains(targetFilename)) {
                            // We can add the file to the Zip Archive
                            if (log.isDebug()) {
                                logDebug(BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip1.Label") + fileList[i] + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip3.Label"));
                            }
                            // Associate a file input stream for the current file
                            InputStream in = KettleVFS.getInputStream(file);
                            // Add ZIP entry to output stream.
                            // 
                            String relativeName;
                            String fullName = fileList[i].getName().getPath();
                            String basePath = sourceFileOrFolder.getName().getPath();
                            if (isSourceDirectory) {
                                if (fullName.startsWith(basePath)) {
                                    relativeName = fullName.substring(basePath.length() + 1);
                                } else {
                                    relativeName = fullName;
                                }
                            } else if (isFromPrevious) {
                                int depth = determineDepth(environmentSubstitute(storedSourcePathDepth));
                                relativeName = determineZipfilenameForDepth(fullName, depth);
                            } else {
                                relativeName = fileList[i].getName().getBaseName();
                            }
                            out.putNextEntry(new ZipEntry(relativeName));
                            int len;
                            while ((len = in.read(buffer)) > 0) {
                                out.write(buffer, 0, len);
                            }
                            out.flush();
                            out.closeEntry();
                            // Close the current file input stream
                            in.close();
                            // Get Zipped File
                            zippedFiles[fileNum] = fileList[i];
                            fileNum = fileNum + 1;
                        }
                    }
                    // Close the ZipOutPutStream
                    out.close();
                    buff.close();
                    dest.close();
                    if (log.isBasic()) {
                        logBasic(BaseMessages.getString(PKG, "JobZipFiles.Log.TotalZippedFiles", "" + zippedFiles.length));
                    }
                    // Delete Temp File
                    if (tempFile != null) {
                        tempFile.delete();
                    }
                    // -----Get the list of Zipped Files and Move or Delete Them
                    if (afterZip == 1 || afterZip == 2) {
                        // iterate through the array of Zipped files
                        for (int i = 0; i < zippedFiles.length; i++) {
                            if (zippedFiles[i] != null) {
                                // Delete, Move File
                                FileObject fileObjectd = zippedFiles[i];
                                if (!isSourceDirectory) {
                                    fileObjectd = KettleVFS.getFileObject(localSourceFilename, this);
                                }
                                // Here we can move, delete files
                                if (afterZip == 1) {
                                    // Delete File
                                    boolean deleted = fileObjectd.delete();
                                    if (!deleted) {
                                        resultat = false;
                                        logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Delete_File1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.Cant_Delete_File2.Label"));
                                    }
                                    // File deleted
                                    if (log.isDebug()) {
                                        logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Deleted1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Deleted2.Label"));
                                    }
                                } else if (afterZip == 2) {
                                    // Move File
                                    FileObject fileObjectm = null;
                                    try {
                                        fileObjectm = KettleVFS.getFileObject(realMovetodirectory + Const.FILE_SEPARATOR + fileObjectd.getName().getBaseName(), this);
                                        fileObjectd.moveTo(fileObjectm);
                                    } catch (IOException e) {
                                        logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File2.Label") + e.getMessage());
                                        resultat = false;
                                    } finally {
                                        try {
                                            if (fileObjectm != null) {
                                                fileObjectm.close();
                                            }
                                        } catch (Exception e) {
                                            if (fileObjectm != null) {
                                                logError("Error closing file '" + fileObjectm.toString() + "'", e);
                                            }
                                        }
                                    }
                                    // File moved
                                    if (log.isDebug()) {
                                        logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Moved1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Moved2.Label"));
                                    }
                                }
                            }
                        }
                    }
                    if (addFileToResult) {
                        // Add file to result files name
                        ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
                        result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
                    }
                    resultat = true;
                }
            }
        } catch (Exception e) {
            logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile2.Label"), e);
            resultat = false;
        } finally {
            if (fileObject != null) {
                try {
                    fileObject.close();
                    fileObject = null;
                } catch (IOException ex) {
                    logError("Error closing file '" + fileObject.toString() + "'", ex);
                }
            }
            try {
                if (out != null) {
                    out.close();
                }
                if (buff != null) {
                    buff.close();
                }
                if (dest != null) {
                    dest.close();
                }
                if (zin != null) {
                    zin.close();
                }
            } catch (IOException ex) {
                logError("Error closing zip file entry for file '" + originFile.toString() + "'", ex);
            }
        }
    } else {
        resultat = false;
        if (localrealZipfilename == null) {
            logError(BaseMessages.getString(PKG, "JobZipFiles.No_ZipFile_Defined.Label"));
        }
        if (!orginExist) {
            logError(BaseMessages.getString(PKG, "JobZipFiles.No_FolderCible_Defined.Label", localSourceFilename));
        }
    }
    // return a verifier
    return resultat;
}

Example 2 with BufferedOutputStreamWithCloseDetection

use of org.pentaho.di.workarounds.BufferedOutputStreamWithCloseDetection in project pentaho-kettle by pentaho.

the class ExcelWriterStep method prepareNextOutputFile.

public void prepareNextOutputFile() throws KettleException {
    try {
        // sheet name shouldn't exceed 31 character
        if (data.realSheetname != null && data.realSheetname.length() > 31) {
            throw new KettleException(BaseMessages.getString(PKG, "ExcelWriterStep.Exception.MaxSheetName", data.realSheetname));
        }
        // clear style cache
        int numOfFields = meta.getOutputFields() != null && meta.getOutputFields().length > 0 ? meta.getOutputFields().length : 0;
        if (numOfFields == 0) {
            numOfFields = data.inputRowMeta != null ? data.inputRowMeta.size() : 0;
        }
        data.clearStyleCache(numOfFields);
        // build new filename
        String buildFilename = buildFilename(data.splitnr);
        data.file = KettleVFS.getFileObject(buildFilename, getTransMeta());
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "ExcelWriterStep.Log.OpeningFile", buildFilename));
        }
        // determine whether existing file must be deleted
        if (data.file.exists() && data.createNewFile) {
            if (!data.file.delete()) {
                if (log.isBasic()) {
                    logBasic(BaseMessages.getString(PKG, "ExcelWriterStep.Log.CouldNotDeleteStaleFile", buildFilename));
                }
                setErrors(1);
                throw new KettleException("Could not delete stale file " + buildFilename);
            }
        }
        // adding filename to result
        if (meta.isAddToResultFiles()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
            resultFile.setComment("This file was created with an Excel writer step by Pentaho Data Integration");
            addResultFile(resultFile);
        }
        boolean appendingToSheet = true;
        // if now no file exists we must create it as indicated by user
        if (!data.file.exists()) {
            // if template file is enabled
            if (meta.isTemplateEnabled()) {
                // handle template case (must have same format)
                // ensure extensions match
                String templateExt = KettleVFS.getFileObject(data.realTemplateFileName).getName().getExtension();
                if (!meta.getExtension().equalsIgnoreCase(templateExt)) {
                    throw new KettleException("Template Format Mismatch: Template has extension: " + templateExt + ", but output file has extension: " + meta.getExtension() + ". Template and output file must share the same format!");
                }
                if (KettleVFS.getFileObject(data.realTemplateFileName).exists()) {
                    // if the template exists just copy the template in place
                    copyFile(KettleVFS.getFileObject(data.realTemplateFileName, getTransMeta()), data.file);
                } else {
                    // template is missing, log it and get out
                    if (log.isBasic()) {
                        logBasic(BaseMessages.getString(PKG, "ExcelWriterStep.Log.TemplateMissing", data.realTemplateFileName));
                    }
                    setErrors(1);
                    throw new KettleException("Template file missing: " + data.realTemplateFileName);
                }
            } else {
                // handle fresh file case, just create a fresh workbook
                Workbook wb = meta.getExtension().equalsIgnoreCase("xlsx") ? new XSSFWorkbook() : new HSSFWorkbook();
                BufferedOutputStreamWithCloseDetection out = new BufferedOutputStreamWithCloseDetection(KettleVFS.getOutputStream(data.file, false));
                wb.createSheet(data.realSheetname);
                wb.write(out);
                out.close();
                wb.close();
            }
            appendingToSheet = false;
        }
        // file is guaranteed to be in place now
        if (meta.getExtension().equalsIgnoreCase("xlsx")) {
            XSSFWorkbook xssfWorkbook = new XSSFWorkbook(KettleVFS.getInputStream(data.file));
            if (meta.isStreamingData() && !meta.isTemplateEnabled()) {
                data.wb = new SXSSFWorkbook(xssfWorkbook, 100);
            } else {
                // Initialize it later after writing header/template because SXSSFWorkbook can't read/rewrite existing data,
                // only append.
                data.wb = xssfWorkbook;
            }
        } else {
            data.wb = new HSSFWorkbook(KettleVFS.getInputStream(data.file));
        }
        int existingActiveSheetIndex = data.wb.getActiveSheetIndex();
        int replacingSheetAt = -1;
        if (data.wb.getSheet(data.realSheetname) != null) {
            // sheet exists, replace or reuse as indicated by user
            if (data.createNewSheet) {
                replacingSheetAt = data.wb.getSheetIndex(data.wb.getSheet(data.realSheetname));
                data.wb.removeSheetAt(replacingSheetAt);
            }
        }
        // if sheet is now missing, we need to create a new one
        if (data.wb.getSheet(data.realSheetname) == null) {
            if (meta.isTemplateSheetEnabled()) {
                Sheet ts = data.wb.getSheet(data.realTemplateSheetName);
                // if template sheet is missing, break
                if (ts == null) {
                    throw new KettleException(BaseMessages.getString(PKG, "ExcelWriterStep.Exception.TemplateNotFound", data.realTemplateSheetName));
                }
                data.sheet = data.wb.cloneSheet(data.wb.getSheetIndex(ts));
                data.wb.setSheetName(data.wb.getSheetIndex(data.sheet), data.realSheetname);
                // unhide sheet in case it was hidden
                data.wb.setSheetHidden(data.wb.getSheetIndex(data.sheet), false);
                if (meta.isTemplateSheetHidden()) {
                    data.wb.setSheetHidden(data.wb.getSheetIndex(ts), true);
                }
            } else {
                // no template to use, simply create a new sheet
                data.sheet = data.wb.createSheet(data.realSheetname);
            }
            if (replacingSheetAt > -1) {
                data.wb.setSheetOrder(data.sheet.getSheetName(), replacingSheetAt);
            }
            // preserves active sheet selection in workbook
            data.wb.setActiveSheet(existingActiveSheetIndex);
            data.wb.setSelectedTab(existingActiveSheetIndex);
            appendingToSheet = false;
        } else {
            // sheet is there and should be reused
            data.sheet = data.wb.getSheet(data.realSheetname);
        }
        // if use chose to make the current sheet active, do so
        if (meta.isMakeSheetActive()) {
            int sheetIndex = data.wb.getSheetIndex(data.sheet);
            data.wb.setActiveSheet(sheetIndex);
            data.wb.setSelectedTab(sheetIndex);
        }
        // handle write protection
        if (meta.isSheetProtected()) {
            protectSheet(data.sheet, data.realPassword);
        }
        // starting cell support
        data.startingRow = 0;
        data.startingCol = 0;
        if (!Utils.isEmpty(data.realStartingCell)) {
            CellReference cellRef = new CellReference(data.realStartingCell);
            data.startingRow = cellRef.getRow();
            data.startingCol = cellRef.getCol();
        }
        data.posX = data.startingCol;
        data.posY = data.startingRow;
        // Find last row and append accordingly
        if (!data.createNewSheet && meta.isAppendLines() && appendingToSheet) {
            data.posY = 0;
            if (data.sheet.getPhysicalNumberOfRows() > 0) {
                data.posY = data.sheet.getLastRowNum();
                data.posY++;
            }
        }
        // Find last row and append accordingly
        if (!data.createNewSheet && meta.getAppendOffset() != 0 && appendingToSheet) {
            data.posY += meta.getAppendOffset();
        }
        // may have to write a few empty lines
        if (!data.createNewSheet && meta.getAppendEmpty() > 0 && appendingToSheet) {
            for (int i = 0; i < meta.getAppendEmpty(); i++) {
                openLine();
                if (!data.shiftExistingCells || meta.isAppendLines()) {
                    data.posY++;
                }
            }
        }
        // may have to write a header here
        if (meta.isHeaderEnabled() && !(!data.createNewSheet && meta.isAppendOmitHeader() && appendingToSheet)) {
            writeHeader();
        }
        if (meta.isStreamingData() && meta.isTemplateEnabled()) {
            data.wb = new SXSSFWorkbook((XSSFWorkbook) data.wb, 100);
            data.sheet = data.wb.getSheet(data.realSheetname);
        }
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "ExcelWriterStep.Log.FileOpened", buildFilename));
        }
        // this is the number of the new output file
        data.splitnr++;
    } catch (Exception e) {
        logError("Error opening new file", e);
        setErrors(1);
        throw new KettleException(e);
    }
}