Examples with VFS org.apache.commons.vfs2.VFS used on opensource projects

Example 21 with VFS

use of org.apache.commons.vfs2.VFS in project metron by apache.

the class VFSClassloaderUtil method configureClassloader.

/**
 * Create a classloader backed by a virtual filesystem which can handle the following URI types:
 * * res - resource files
 * * jar
 * * tar
 * * bz2
 * * tgz
 * * zip
 * * HDFS
 * * FTP
 * * HTTP/S
 * * file
 * @param paths A set of comma separated paths.  The paths are URIs or URIs with a regex pattern at the end.
 * @return A classloader object if it can create it
 * @throws FileSystemException
 */
public static Optional<ClassLoader> configureClassloader(String paths) throws FileSystemException {
    if (paths.trim().isEmpty()) {
        return Optional.empty();
    }
    FileSystemManager vfs = generateVfs();
    FileObject[] objects = resolve(vfs, paths);
    if (objects == null || objects.length == 0) {
        return Optional.empty();
    }
    return Optional.of(new VFSClassLoader(objects, vfs, vfs.getClass().getClassLoader()));
}

Example 22 with VFS

use of org.apache.commons.vfs2.VFS in project wso2-synapse by wso2.

the class PollTableEntry method loadConfiguration.

@Override
public boolean loadConfiguration(ParameterInclude params) throws AxisFault {
    resolveHostsDynamically = ParamUtils.getOptionalParamBoolean(params, VFSConstants.TRANSPORT_FILE_RESOLVEHOST_DYNAMICALLY, false);
    fileURI = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_FILE_URI);
    fileURI = decryptIfRequired(fileURI);
    if (fileURI == null) {
        log.warn("transport.vfs.FileURI parameter is missing in the proxy service configuration");
        return false;
    } else {
        if (fileURI.startsWith(VFSConstants.VFS_PREFIX)) {
            fileURI = fileURI.substring(VFSConstants.VFS_PREFIX.length());
        }
        fileURI = resolveHostAtDeployment(fileURI);
        replyFileURI = ParamUtils.getOptionalParam(params, VFSConstants.REPLY_FILE_URI);
        replyFileURI = decryptIfRequired(replyFileURI);
        replyFileURI = resolveHostAtDeployment(replyFileURI);
        fileNamePattern = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_FILE_NAME_PATTERN);
        contentType = ParamUtils.getRequiredParam(params, VFSConstants.TRANSPORT_FILE_CONTENT_TYPE);
        String option = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_ACTION_AFTER_PROCESS);
        if (option == null) {
            option = VFSTransportListener.DELETE;
        }
        switch(option) {
            case VFSTransportListener.MOVE:
                actionAfterProcess = PollTableEntry.MOVE;
                break;
            case VFSTransportListener.DELETE:
                actionAfterProcess = PollTableEntry.DELETE;
                break;
            case VFSTransportListener.NONE:
                actionAfterProcess = PollTableEntry.NONE;
                break;
            default:
                actionAfterProcess = PollTableEntry.DELETE;
        }
        option = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_ACTION_AFTER_ERRORS);
        actionAfterErrors = VFSTransportListener.MOVE.equals(option) ? PollTableEntry.MOVE : PollTableEntry.DELETE;
        option = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_ACTION_AFTER_FAILURE);
        if (option == null) {
            option = VFSTransportListener.DELETE;
        }
        switch(option) {
            case VFSTransportListener.MOVE:
                actionAfterFailure = PollTableEntry.MOVE;
                break;
            case VFSTransportListener.DELETE:
                actionAfterFailure = PollTableEntry.DELETE;
                break;
            case VFSTransportListener.NONE:
                actionAfterFailure = PollTableEntry.NONE;
                break;
            default:
                actionAfterFailure = PollTableEntry.DELETE;
        }
        String moveDirectoryAfterProcess = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MOVE_AFTER_PROCESS);
        moveDirectoryAfterProcess = decryptIfRequired(moveDirectoryAfterProcess);
        setMoveAfterProcess(moveDirectoryAfterProcess);
        String moveDirectoryAfterErrors = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MOVE_AFTER_ERRORS);
        moveDirectoryAfterErrors = decryptIfRequired(moveDirectoryAfterErrors);
        setMoveAfterErrors(moveDirectoryAfterErrors);
        String moveDirectoryAfterFailure = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MOVE_AFTER_FAILURE);
        moveDirectoryAfterFailure = decryptIfRequired(moveDirectoryAfterFailure);
        setMoveAfterFailure(moveDirectoryAfterFailure);
        String moveFileTimestampFormat = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MOVE_TIMESTAMP_FORMAT);
        if (moveFileTimestampFormat != null) {
            moveTimestampFormat = new SimpleDateFormat(moveFileTimestampFormat);
        }
        setVfsSchemeProperties(VFSUtils.parseSchemeFileOptions(fileURI, params));
        String strStreaming = ParamUtils.getOptionalParam(params, VFSConstants.STREAMING);
        if (strStreaming != null) {
            streaming = Boolean.parseBoolean(strStreaming);
        }
        String strMaxRetryCount = ParamUtils.getOptionalParam(params, VFSConstants.MAX_RETRY_COUNT);
        maxRetryCount = strMaxRetryCount != null ? Integer.parseInt(strMaxRetryCount) : VFSConstants.DEFAULT_MAX_RETRY_COUNT;
        String strReconnectTimeout = ParamUtils.getOptionalParam(params, VFSConstants.RECONNECT_TIMEOUT);
        reconnectTimeout = strReconnectTimeout != null ? Integer.parseInt(strReconnectTimeout) * 1000 : VFSConstants.DEFAULT_RECONNECT_TIMEOUT;
        String strFileLocking = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_LOCKING);
        if (VFSConstants.TRANSPORT_FILE_LOCKING_ENABLED.equals(strFileLocking)) {
            fileLocking = true;
        } else if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strFileLocking)) {
            fileLocking = false;
        }
        String strFileSizeLimit = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_SIZE_LIMIT);
        try {
            fileSizeLimit = strFileSizeLimit != null ? Double.parseDouble(strFileSizeLimit) : VFSConstants.DEFAULT_TRANSPORT_FILE_SIZE_LIMIT;
        } catch (Exception e) {
            log.warn("Error parsing specified file size limit - " + strFileSizeLimit + ", using default - unlimited");
        }
        moveAfterMoveFailure = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MOVE_AFTER_FAILED_MOVE);
        moveAfterMoveFailure = decryptIfRequired(moveAfterMoveFailure);
        moveAfterMoveFailure = resolveHostAtDeployment(moveAfterMoveFailure);
        String nextRetryDuration = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FAILED_RECORD_NEXT_RETRY_DURATION);
        nextRetryDurationForFailedMove = nextRetryDuration != null ? Integer.parseInt(nextRetryDuration) : VFSConstants.DEFAULT_NEXT_RETRY_DURATION;
        failedRecordFileName = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FAILED_RECORDS_FILE_NAME);
        if (failedRecordFileName == null) {
            failedRecordFileName = VFSConstants.DEFAULT_FAILED_RECORDS_FILE_NAME;
        }
        failedRecordFileDestination = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FAILED_RECORDS_FILE_DESTINATION);
        if (failedRecordFileDestination == null) {
            failedRecordFileDestination = VFSConstants.DEFAULT_FAILED_RECORDS_FILE_DESTINATION;
        }
        failedRecordTimestampFormat = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FAILED_RECORD_TIMESTAMP_FORMAT);
        if (failedRecordTimestampFormat == null) {
            failedRecordTimestampFormat = VFSConstants.DEFAULT_TRANSPORT_FAILED_RECORD_TIMESTAMP_FORMAT;
        }
        String strFileProcessingInterval = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_INTERVAL);
        fileProcessingInterval = null;
        if (strFileProcessingInterval != null) {
            try {
                fileProcessingInterval = Integer.parseInt(strFileProcessingInterval);
            } catch (NumberFormatException nfe) {
                log.warn("VFS File Processing Interval not set correctly. Current value is : " + strFileProcessingInterval, nfe);
            }
        }
        String strFileProcessingCount = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_COUNT);
        fileProcessingCount = null;
        if (strFileProcessingCount != null) {
            try {
                fileProcessingCount = Integer.parseInt(strFileProcessingCount);
            } catch (NumberFormatException nfe) {
                log.warn("VFS File Processing Count not set correctly. Current value is : " + strFileProcessingCount, nfe);
            }
        }
        String strAutoLock = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE);
        autoLockRelease = false;
        autoLockReleaseSameNode = true;
        autoLockReleaseInterval = null;
        if (strAutoLock != null) {
            try {
                autoLockRelease = Boolean.parseBoolean(strAutoLock);
            } catch (Exception e) {
                autoLockRelease = false;
                log.warn("VFS Auto lock removal not set properly. Current value is : " + strAutoLock, e);
            }
            if (autoLockRelease) {
                String strAutoLockInterval = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE_INTERVAL);
                if (strAutoLockInterval != null) {
                    try {
                        autoLockReleaseInterval = Long.parseLong(strAutoLockInterval);
                    } catch (Exception e) {
                        autoLockReleaseInterval = null;
                        log.warn("VFS Auto lock removal property not set properly. Current value is : " + strAutoLockInterval, e);
                    }
                }
                String strAutoLockReleaseSameNode = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE_SAME_NODE);
                if (strAutoLockReleaseSameNode != null) {
                    try {
                        autoLockReleaseSameNode = Boolean.parseBoolean(strAutoLockReleaseSameNode);
                    } catch (Exception e) {
                        autoLockReleaseSameNode = true;
                        log.warn("VFS Auto lock removal property not set properly. Current value is : " + autoLockReleaseSameNode, e);
                    }
                }
            }
        }
        distributedLock = false;
        distributedLockTimeout = null;
        String strDistributedLock = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_DISTRIBUTED_LOCK);
        if (strDistributedLock != null) {
            try {
                distributedLock = Boolean.parseBoolean(strDistributedLock);
            } catch (Exception e) {
                autoLockRelease = false;
                log.warn("VFS Distributed lock not set properly. Current value is : " + strDistributedLock, e);
            }
            if (distributedLock) {
                String strDistributedLockTimeout = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_DISTRIBUTED_LOCK_TIMEOUT);
                if (strDistributedLockTimeout != null) {
                    try {
                        distributedLockTimeout = Long.parseLong(strDistributedLockTimeout);
                    } catch (Exception e) {
                        distributedLockTimeout = null;
                        log.warn("VFS Distributed lock timeout property not set properly. Current value is : " + strDistributedLockTimeout, e);
                    }
                }
            }
        }
        fileSortParam = ParamUtils.getOptionalParam(params, VFSConstants.FILE_SORT_PARAM);
        fileSortAscending = true;
        if (fileSortParam != null && ParamUtils.getOptionalParam(params, VFSConstants.FILE_SORT_ORDER) != null) {
            try {
                fileSortAscending = Boolean.parseBoolean(ParamUtils.getOptionalParam(params, VFSConstants.FILE_SORT_ORDER));
            } catch (Exception e) {
                fileSortAscending = true;
            }
        }
        String strForceCreateFolder = ParamUtils.getOptionalParam(params, VFSConstants.FORCE_CREATE_FOLDER);
        forceCreateFolder = false;
        if (strForceCreateFolder != null && "true".equals(strForceCreateFolder.toLowerCase())) {
            forceCreateFolder = true;
        }
        subfolderTimestamp = ParamUtils.getOptionalParam(params, VFSConstants.SUBFOLDER_TIMESTAMP);
        this.clusterAware = ParamUtils.getOptionalParamBoolean(params, VFSConstants.CLUSTER_AWARE, false);
        return super.loadConfiguration(params);
    }
}

Example 23 with VFS

use of org.apache.commons.vfs2.VFS in project wso2-synapse by wso2.

the class VFSTransportListener method scanFileOrDirectory.

/**
 * Search for files that match the given regex pattern and create a list
 * Then process each of these files and update the status of the scan on
 * the poll table
 * @param entry the poll table entry for the scan
 * @param fileURI the file or directory to be scanned
 */
private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {
    if (log.isDebugEnabled()) {
        log.debug("Polling: " + VFSUtils.maskURLPassword(fileURI));
    }
    if (entry.isClusterAware()) {
        boolean leader = true;
        ClusteringAgent agent = getConfigurationContext().getAxisConfiguration().getClusteringAgent();
        if (agent != null && agent.getParameter("domain") != null) {
            // hazelcast clustering instance name
            String hazelcastInstanceName = agent.getParameter("domain").getValue() + ".instance";
            HazelcastInstance instance = Hazelcast.getHazelcastInstanceByName(hazelcastInstanceName);
            if (instance != null) {
                // dirty leader election
                leader = instance.getCluster().getMembers().iterator().next().localMember();
            } else {
                log.warn("Clustering error, running the polling task in this node");
            }
        } else {
            log.warn("Although proxy is cluster aware, clustering config are not present, hence running the" + " the polling task in this node");
        }
        if (!leader) {
            if (log.isDebugEnabled()) {
                log.debug("This Member is not the leader");
            }
            entry.setLastPollState(PollTableEntry.NONE);
            long now = System.currentTimeMillis();
            entry.setLastPollTime(now);
            entry.setNextPollTime(now + entry.getPollInterval());
            onPollCompletion(entry);
            return;
        }
        if (log.isDebugEnabled()) {
            log.debug("This Member is the leader");
        }
    }
    FileSystemOptions fso = null;
    setFileSystemClosed(false);
    try {
        fso = VFSUtils.attachFileSystemOptions(entry.getVfsSchemeProperties(), fsManager);
    } catch (Exception e) {
        log.error("Error while attaching VFS file system properties. " + e.getMessage());
    }
    FileObject fileObject = null;
    // TODO : Trying to make the correct URL out of the malformed one.
    if (fileURI.contains("vfs:")) {
        fileURI = fileURI.substring(fileURI.indexOf("vfs:") + 4);
    }
    if (log.isDebugEnabled()) {
        log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
    }
    boolean wasError = true;
    int retryCount = 0;
    int maxRetryCount = entry.getMaxRetryCount();
    long reconnectionTimeout = entry.getReconnectTimeout();
    while (wasError) {
        try {
            retryCount++;
            fileObject = fsManager.resolveFile(fileURI, fso);
            if (fileObject == null) {
                log.error("fileObject is null");
                throw new FileSystemException("fileObject is null");
            }
            wasError = false;
        } catch (FileSystemException e) {
            if (retryCount >= maxRetryCount) {
                processFailure("Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry);
                closeFileSystem(fileObject);
                return;
            } else {
                log.warn("Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds.");
            }
        }
        if (wasError) {
            try {
                Thread.sleep(reconnectionTimeout);
            } catch (InterruptedException e2) {
                log.error("Thread was interrupted while waiting to reconnect.", e2);
            }
        }
    }
    try {
        if (fileObject.exists() && fileObject.isReadable()) {
            entry.setLastPollState(PollTableEntry.NONE);
            FileObject[] children = null;
            try {
                children = fileObject.getChildren();
            } catch (FileNotFolderException ignored) {
            } catch (FileSystemException ex) {
                log.error(ex.getMessage(), ex);
            }
            // if this is a file that would translate to a single message
            if (children == null || children.length == 0) {
                boolean isFailedRecord = false;
                if (entry.getMoveAfterMoveFailure() != null) {
                    isFailedRecord = isFailedRecord(fileObject, entry);
                }
                if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
                    boolean runPostProcess = true;
                    if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && acquireLock(fsManager, fileObject, entry, fso, true))) {
                        try {
                            if (fileObject.getType() == FileType.FILE) {
                                processFile(entry, fileObject);
                                entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                                metrics.incrementMessagesReceived();
                            } else {
                                runPostProcess = false;
                            }
                        } catch (AxisFault e) {
                            if (e.getCause() instanceof FileNotFoundException) {
                                log.warn("Error processing File URI : " + VFSUtils.maskURLPassword(fileObject.getName().toString()) + ". This can be due to file moved from another process.");
                                runPostProcess = false;
                            } else {
                                logException("Error processing File URI : " + VFSUtils.maskURLPassword(fileObject.getName().getURI()), e);
                                entry.setLastPollState(PollTableEntry.FAILED);
                                metrics.incrementFaultsReceiving();
                            }
                        }
                        if (runPostProcess) {
                            try {
                                moveOrDeleteAfterProcessing(entry, fileObject, fso);
                            } catch (AxisFault axisFault) {
                                logException("File object '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' " + "cloud not be moved", axisFault);
                                entry.setLastPollState(PollTableEntry.FAILED);
                                String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                                addFailedRecord(entry, fileObject, timeStamp);
                            }
                        }
                        if (entry.isFileLockingEnabled()) {
                            VFSUtils.releaseLock(fsManager, fileObject, fso);
                            if (log.isDebugEnabled()) {
                                log.debug("Removed the lock file '" + VFSUtils.maskURLPassword(fileObject.toString()) + ".lock' of the file '" + VFSUtils.maskURLPassword(fileObject.toString()));
                            }
                        }
                    } else if (log.isDebugEnabled()) {
                        log.debug("Couldn't get the lock for processing the file : " + VFSUtils.maskURLPassword(fileObject.getName().getURI()));
                    } else if (isFailedRecord) {
                        if (entry.isFileLockingEnabled()) {
                            VFSUtils.releaseLock(fsManager, fileObject, fso);
                        }
                        // schedule a cleanup task if the file is there
                        if (fsManager.resolveFile(fileObject.getURL().toString(), fso) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) {
                            workerPool.execute(new FileRemoveTask(entry, fileObject, fso));
                        }
                        if (log.isDebugEnabled()) {
                            log.debug("File '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' has been marked as a failed" + " record, it will not process");
                        }
                    }
                }
            } else {
                int failCount = 0;
                int successCount = 0;
                int processCount = 0;
                Integer iFileProcessingInterval = entry.getFileProcessingInterval();
                Integer iFileProcessingCount = entry.getFileProcessingCount();
                if (log.isDebugEnabled()) {
                    log.debug("File name pattern : " + entry.getFileNamePattern());
                }
                // Sort the files
                String strSortParam = entry.getFileSortParam();
                if (strSortParam != null) {
                    log.debug("Start Sorting the files.");
                    boolean bSortOrderAsscending = entry.isFileSortAscending();
                    if (log.isDebugEnabled()) {
                        log.debug("Sorting the files by : " + strSortParam + ". (" + bSortOrderAsscending + ")");
                    }
                    if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && bSortOrderAsscending) {
                        Arrays.sort(children, new FileNameAscComparator());
                    } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && !bSortOrderAsscending) {
                        Arrays.sort(children, new FileNameDesComparator());
                    } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && bSortOrderAsscending) {
                        Arrays.sort(children, new FileSizeAscComparator());
                    } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && !bSortOrderAsscending) {
                        Arrays.sort(children, new FileSizeDesComparator());
                    } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && bSortOrderAsscending) {
                        Arrays.sort(children, new FileLastmodifiedtimestampAscComparator());
                    } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && !bSortOrderAsscending) {
                        Arrays.sort(children, new FileLastmodifiedtimestampDesComparator());
                    }
                    log.debug("End Sorting the files.");
                }
                for (FileObject child : children) {
                    // Stop processing when service get undeployed
                    if (state != BaseConstants.STARTED || !entry.getService().isActive()) {
                        return;
                    }
                    /**
                     * Before starting to process another file, see whether the proxy is stopped or not.
                     */
                    if (entry.isCanceled()) {
                        break;
                    }
                    // skipping *.lock file
                    if (child.getName().getBaseName().endsWith(".lock")) {
                        continue;
                    }
                    // skipping subfolders
                    if (child.getType() != FileType.FILE) {
                        continue;
                    }
                    // skipping files depending on size limitation
                    if (entry.getFileSizeLimit() >= 0 && child.getContent().getSize() > entry.getFileSizeLimit()) {
                        if (log.isDebugEnabled()) {
                            log.debug("Ignoring file - " + child.getName().getBaseName() + " size - " + child.getContent().getSize() + " since it exceeds file size limit - " + entry.getFileSizeLimit());
                        }
                        continue;
                    }
                    boolean isFailedRecord = false;
                    if (entry.getMoveAfterMoveFailure() != null) {
                        isFailedRecord = isFailedRecord(child, entry);
                    }
                    if (entry.getFileNamePattern() != null && child.getName().getBaseName().matches(entry.getFileNamePattern())) {
                        // now we try to get the lock and process
                        if (log.isDebugEnabled()) {
                            log.debug("Matching file : " + child.getName().getBaseName());
                        }
                        boolean runPostProcess = true;
                        if ((!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child, fso, true))) && !isFailedRecord) {
                            // process the file
                            try {
                                if (log.isDebugEnabled()) {
                                    log.debug("Processing file :" + VFSUtils.maskURLPassword(child.toString()));
                                }
                                processCount++;
                                if (child.getType() == FileType.FILE) {
                                    processFile(entry, child);
                                    successCount++;
                                    // tell moveOrDeleteAfterProcessing() file was success
                                    entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                                    metrics.incrementMessagesReceived();
                                } else {
                                    runPostProcess = false;
                                }
                            } catch (Exception e) {
                                if (e.getCause() instanceof FileNotFoundException) {
                                    log.warn("Error processing File URI : " + VFSUtils.maskURLPassword(child.getName().toString()) + ". This can be due to file moved from another process.");
                                    runPostProcess = false;
                                } else {
                                    logException("Error processing File URI : " + VFSUtils.maskURLPassword(child.getName().getURI()), e);
                                    failCount++;
                                    // tell moveOrDeleteAfterProcessing() file failed
                                    entry.setLastPollState(PollTableEntry.FAILED);
                                    metrics.incrementFaultsReceiving();
                                }
                            }
                            // skipping un-locking file if failed to do delete/move after process
                            boolean skipUnlock = false;
                            if (runPostProcess) {
                                try {
                                    moveOrDeleteAfterProcessing(entry, child, fso);
                                } catch (AxisFault axisFault) {
                                    logException("File object '" + VFSUtils.maskURLPassword(child.getURL().toString()) + "'cloud not be moved, will remain in \"locked\" state", axisFault);
                                    skipUnlock = true;
                                    failCount++;
                                    entry.setLastPollState(PollTableEntry.FAILED);
                                    String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                                    addFailedRecord(entry, child, timeStamp);
                                }
                            }
                            // if there is a failure or not we'll try to release the lock
                            if (entry.isFileLockingEnabled() && !skipUnlock) {
                                VFSUtils.releaseLock(fsManager, child, fso);
                            }
                        }
                    } else if (entry.getFileNamePattern() != null && !child.getName().getBaseName().matches(entry.getFileNamePattern())) {
                        // child's file name does not match the file name pattern
                        if (log.isDebugEnabled()) {
                            log.debug("Non-Matching file : " + child.getName().getBaseName());
                        }
                    } else if (isFailedRecord) {
                        // it is a failed record
                        if (entry.isFileLockingEnabled()) {
                            VFSUtils.releaseLock(fsManager, child, fso);
                            VFSUtils.releaseLock(fsManager, fileObject, fso);
                        }
                        if (fsManager.resolveFile(child.getURL().toString(), fso) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) {
                            workerPool.execute(new FileRemoveTask(entry, child, fso));
                        }
                        if (log.isDebugEnabled()) {
                            log.debug("File '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' has been marked as a failed record, it will not " + "process");
                        }
                    }
                    if (iFileProcessingInterval != null && iFileProcessingInterval > 0) {
                        try {
                            if (log.isDebugEnabled()) {
                                log.debug("Put the VFS processor to sleep for : " + iFileProcessingInterval);
                            }
                            Thread.sleep(iFileProcessingInterval);
                        } catch (InterruptedException ie) {
                            log.error("Unable to set the interval between file processors." + ie);
                        }
                    } else if (iFileProcessingCount != null && iFileProcessingCount <= processCount) {
                        break;
                    }
                }
                if (failCount == 0 && successCount > 0) {
                    entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                } else if (successCount == 0 && failCount > 0) {
                    entry.setLastPollState(PollTableEntry.FAILED);
                } else {
                    entry.setLastPollState(PollTableEntry.WITH_ERRORS);
                }
            }
            // processing of this poll table entry is complete
            long now = System.currentTimeMillis();
            entry.setLastPollTime(now);
            entry.setNextPollTime(now + entry.getPollInterval());
        } else if (log.isDebugEnabled()) {
            log.debug("Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!"));
        }
        onPollCompletion(entry);
    } catch (FileSystemException e) {
        processFailure("Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry);
    } catch (Exception ex) {
        processFailure("Un-handled exception thrown when processing the file : ", ex, entry);
    } finally {
        closeFileSystem(fileObject);
    }
}

Example 24 with VFS

use of org.apache.commons.vfs2.VFS in project wso2-synapse by wso2.

the class VFSTransportSender method init.

/**
 * Initialize the VFS file system manager and be ready to send messages
 * @param cfgCtx the axis2 configuration context
 * @param transportOut the transport-out description
 * @throws AxisFault on error
 */
public void init(ConfigurationContext cfgCtx, TransportOutDescription transportOut) throws AxisFault {
    super.init(cfgCtx, transportOut);
    try {
        StandardFileSystemManager fsm = new StandardFileSystemManager();
        fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml"));
        fsm.init();
        fsManager = fsm;
        Parameter lckFlagParam = transportOut.getParameter(VFSConstants.TRANSPORT_FILE_LOCKING);
        if (lckFlagParam != null) {
            String strLockingFlag = lckFlagParam.getValue().toString();
            // by-default enabled, if explicitly specified as "disable" make it disable
            if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strLockingFlag)) {
                globalFileLockingFlag = false;
            }
        }
        Parameter strAutoLock = transportOut.getParameter(VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE);
        boolean autoLockRelease = false;
        boolean autoLockReleaseSameNode = true;
        Long autoLockReleaseInterval = null;
        if (strAutoLock != null && strAutoLock.getValue() != null && !strAutoLock.getValue().toString().isEmpty()) {
            try {
                autoLockRelease = Boolean.parseBoolean(strAutoLock.getValue().toString());
            } catch (Exception e) {
                autoLockRelease = false;
                log.warn("VFS Auto lock removal not set properly. Given value is : " + strAutoLock + ", defaults to - " + autoLockRelease, e);
            }
            if (autoLockRelease) {
                Parameter strAutoLockInterval = transportOut.getParameter(VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE_INTERVAL);
                if (strAutoLockInterval != null && strAutoLockInterval.getValue() != null && !strAutoLockInterval.getValue().toString().isEmpty()) {
                    try {
                        autoLockReleaseInterval = Long.parseLong(strAutoLockInterval.getValue().toString());
                    } catch (Exception e) {
                        autoLockReleaseInterval = null;
                        log.warn("VFS Auto lock release interval is not set properly. Given value is : " + strAutoLockInterval + ", defaults to - null", e);
                    }
                }
                Parameter strAutoLockReleaseSameNode = transportOut.getParameter(VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE_SAME_NODE);
                if (strAutoLockReleaseSameNode != null && strAutoLockReleaseSameNode.getValue() != null && !strAutoLockReleaseSameNode.getValue().toString().isEmpty()) {
                    try {
                        autoLockReleaseSameNode = Boolean.parseBoolean(strAutoLockReleaseSameNode.getValue().toString());
                    } catch (Exception e) {
                        autoLockReleaseSameNode = true;
                        log.warn("VFS Auto lock removal same node property not set properly. Given value is : " + autoLockReleaseSameNode + ", defaults to - " + autoLockReleaseSameNode, e);
                    }
                }
            }
        }
        vfsParamDTO = new VFSParamDTO();
        vfsParamDTO.setAutoLockRelease(autoLockRelease);
        vfsParamDTO.setAutoLockReleaseInterval(autoLockReleaseInterval);
        vfsParamDTO.setAutoLockReleaseSameNode(autoLockReleaseSameNode);
    } catch (FileSystemException e) {
        handleException("Error initializing the file transport : " + e.getMessage(), e);
    }
}

Example 25 with VFS

use of org.apache.commons.vfs2.VFS in project accumulo by apache.

the class AccumuloVFSClassLoader method getClassLoader.

public static ClassLoader getClassLoader() throws IOException {
    ReloadingClassLoader localLoader = loader;
    while (null == localLoader) {
        synchronized (lock) {
            if (null == loader) {
                FileSystemManager vfs = generateVfs();
                // Set up the 2nd tier class loader
                if (null == parent) {
                    parent = AccumuloClassLoader.getClassLoader();
                }
                FileObject[] vfsCP = resolve(vfs, AccumuloClassLoader.getAccumuloProperty(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY, ""));
                if (vfsCP.length == 0) {
                    localLoader = createDynamicClassloader(parent);
                    loader = localLoader;
                    return localLoader.getClassLoader();
                }
                // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
                localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
                loader = localLoader;
                // and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration)
                for (FileObject fo : vfsCP) {
                    if (fo instanceof HdfsFileObject) {
                        String uri = fo.getName().getRootURI();
                        Configuration c = new Configuration(true);
                        c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
                        FileSystem fs = FileSystem.get(c);
                        fs.getConf().setClassLoader(loader.getClassLoader());
                    }
                }
            }
        }
    }
    return localLoader.getClassLoader();
}