Examples with HopFileException org.apache.hop.core.exception.HopFileException used on opensource projects

Example 11 with HopFileException

use of org.apache.hop.core.exception.HopFileException in project hop by apache.

the class GroupBy method addToBuffer.

// Method is defined as package-protected in order to be accessible by unit tests
void addToBuffer(Object[] row) throws HopFileException {
    data.bufferList.add(row);
    if (data.bufferList.size() > 5000 && data.rowsOnFile == 0) {
        String pathToTmp = resolve(getMeta().getDirectory());
        try {
            File ioFile = new File(pathToTmp);
            if (!ioFile.exists()) {
                // try to resolve as Apache VFS file
                pathToTmp = retrieveVfsPath(pathToTmp);
            }
            data.tempFile = File.createTempFile(getMeta().getPrefix(), ".tmp", new File(pathToTmp));
            data.fosToTempFile = new FileOutputStream(data.tempFile);
            data.dosToTempFile = new DataOutputStream(data.fosToTempFile);
            data.firstRead = true;
        } catch (IOException e) {
            throw new HopFileException(BaseMessages.getString(PKG, "GroupBy.Exception.UnableToCreateTemporaryFile"), e);
        }
        // OK, save the oldest rows to disk!
        Object[] oldest = data.bufferList.get(0);
        data.inputRowMeta.writeData(data.dosToTempFile, oldest);
        data.bufferList.remove(0);
        data.rowsOnFile++;
    }
}

Example 12 with HopFileException

use of org.apache.hop.core.exception.HopFileException in project hop by apache.

the class GitGuiPlugin method filePainted.

/**
 * If we have a git project we can take a look and see if a file is changed.
 *
 * @param tree
 * @param treeItem
 * @param path
 * @param name
 */
@Override
public void filePainted(Tree tree, TreeItem treeItem, String path, String name) {
    GuiResource guiResource = GuiResource.getInstance();
    UIFile file = null;
    // Changed git file colored blue
    try {
        file = changedFiles.get(HopVfs.getFileObject(path).getName().getPath());
    } catch (HopFileException e) {
    // do nothing
    }
    if (file != null) {
        switch(file.getChangeType()) {
            case DELETE:
            case MODIFY:
            case RENAME:
            case COPY:
                treeItem.setForeground(colorStaged);
                break;
            case ADD:
                if (file.getIsStaged()) {
                    treeItem.setForeground(colorStaged);
                } else {
                    treeItem.setForeground(colorUnstaged);
                }
                break;
        }
    }
    String ignored = ignoredFiles.get(path);
    if (ignored != null) {
        treeItem.setForeground(colorIgnored);
    }
}

Example 13 with HopFileException

use of org.apache.hop.core.exception.HopFileException in project hop by apache.

the class FileMetadata method buildOutputRows.

private void buildOutputRows() throws HopTransformException {
    // which index does the next field go to
    int idx = data.isReceivingInput ? getInputRowMeta().size() : 0;
    // prepare an output row
    Object[] outputRow = data.isReceivingInput ? RowDataUtil.createResizedCopy(r, data.outputRowMeta.size()) : RowDataUtil.allocateRowData(data.outputRowMeta.size());
    // get the configuration from the dialog
    String fileName = resolve(meta.getFileName());
    // if the file does not exist, just send an empty row
    try {
        if (!HopVfs.fileExists(fileName)) {
            putRow(data.outputRowMeta, outputRow);
            return;
        }
    } catch (HopFileException e) {
        throw new HopTransformException(e.getMessage(), e);
    }
    String strLimitRows = resolve(meta.getLimitRows());
    if (strLimitRows.trim().isEmpty()) {
        limitRows = 0;
    } else {
        limitRows = Long.parseLong(strLimitRows);
    }
    defaultCharset = Charset.forName(resolve(meta.getDefaultCharset()));
    ArrayList<Character> delimiterCandidates = new ArrayList<>(4);
    for (String candidate : meta.getDelimiterCandidates()) {
        candidate = resolve(candidate);
        if (candidate.length() == 0) {
            logBasic("Warning: file metadata transform ignores empty delimiter candidate");
        } else if (candidate.length() > 1) {
            logBasic("Warning: file metadata transform ignores non-character delimiter candidate: " + candidate);
        } else {
            delimiterCandidates.add(candidate.charAt(0));
        }
    }
    ArrayList<Character> enclosureCandidates = new ArrayList<>(4);
    for (String candidate : meta.getEnclosureCandidates()) {
        candidate = resolve(candidate);
        if (candidate.length() == 0) {
            logBasic("Warning: file metadata transform ignores empty enclosure candidate");
        } else if (candidate.length() > 1) {
            logBasic("Warning: file metadata transform ignores non-character enclosure candidate: " + candidate);
        } else {
            enclosureCandidates.add(candidate.charAt(0));
        }
    }
    // guess the charset
    Charset detectedCharset = detectCharset(fileName);
    outputRow[idx++] = detectedCharset;
    // guess the delimiters
    DelimiterDetector.DetectionResult delimiters = detectDelimiters(fileName, detectedCharset, delimiterCandidates, enclosureCandidates);
    if (delimiters == null) {
        throw new HopTransformException("Could not determine a consistent format for file " + fileName);
    }
    // delimiter
    outputRow[idx++] = delimiters.getDelimiter();
    // enclosure
    outputRow[idx++] = delimiters.getEnclosure() == null ? "" : delimiters.getEnclosure().toString();
    // field count = delimiter frequency on data lines +1
    outputRow[idx++] = delimiters.getDataLineFrequency() + 1L;
    // bad headers
    outputRow[idx++] = delimiters.getBadHeaders();
    // bad footers
    outputRow[idx++] = delimiters.getBadFooters();
    char delimiter = delimiters.getDelimiter();
    char enclosure = delimiters.getEnclosure() == null ? '\u0000' : delimiters.getEnclosure();
    long skipLines = delimiters.getBadHeaders();
    long dataLines = delimiters.getDataLines();
    try (BufferedReader inputReader = new BufferedReader(new InputStreamReader(HopVfs.getInputStream(fileName), detectedCharset))) {
        while (skipLines > 0) {
            skipLines--;
            inputReader.readLine();
        }
        CSVReader csvReader = new CSVReader(inputReader, delimiter, enclosure);
        String[] firstLine = csvReader.readNext();
        dataLines--;
        StringEvaluator[] evaluators = new StringEvaluator[firstLine.length];
        for (int i = 0; i < evaluators.length; i++) {
            evaluators[i] = new StringEvaluator(true);
        }
        while (dataLines > 0) {
            dataLines--;
            String[] fields = csvReader.readNext();
            if (fields == null)
                break;
            for (int i = 0; i < fields.length; i++) {
                if (i < evaluators.length)
                    evaluators[i].evaluateString(fields[i]);
            }
        }
        // find evaluation results, excluding and including the first line
        IValueMeta[] fields = new IValueMeta[evaluators.length];
        IValueMeta[] firstLineFields = new IValueMeta[evaluators.length];
        for (int i = 0; i < evaluators.length; i++) {
            fields[i] = evaluators[i].getAdvicedResult().getConversionMeta();
            evaluators[i].evaluateString(firstLine[i]);
            firstLineFields[i] = evaluators[i].getAdvicedResult().getConversionMeta();
        }
        // check whether to use the first line as a header, if there is a single type mismatch -> yes
        // if all fields are strings -> yes
        boolean hasHeader = false;
        boolean allStrings = true;
        for (int i = 0; i < evaluators.length; i++) {
            if (fields[i].getType() != IValueMeta.TYPE_STRING) {
                allStrings = false;
            }
            if (fields[i].getType() != firstLineFields[i].getType()) {
                hasHeader = true;
                break;
            }
        }
        hasHeader = hasHeader || allStrings;
        if (hasHeader) {
            for (int i = 0; i < evaluators.length; i++) {
                fields[i].setName(firstLine[i].trim());
            }
        } else {
            // use the meta from the entire column
            fields = firstLineFields;
            int colNum = 1;
            for (int i = 0; i < evaluators.length; i++) {
                fields[i].setName("field_" + (colNum++));
            }
        }
        outputRow[idx++] = hasHeader;
        int fieldIdx = idx;
        for (int i = 0; i < evaluators.length; i++) {
            outputRow = RowDataUtil.createResizedCopy(outputRow, outputRow.length);
            idx = fieldIdx;
            outputRow[idx++] = fields[i].getName();
            outputRow[idx++] = fields[i].getTypeDesc();
            outputRow[idx++] = (fields[i].getLength() >= 0) ? (long) fields[i].getLength() : null;
            outputRow[idx++] = (fields[i].getPrecision() >= 0) ? (long) fields[i].getPrecision() : null;
            outputRow[idx++] = fields[i].getConversionMask();
            outputRow[idx++] = fields[i].getDecimalSymbol();
            outputRow[idx++] = fields[i].getGroupingSymbol();
            putRow(data.outputRowMeta, outputRow);
        }
    } catch (IOException | HopFileException e) {
        log.logError("IO Error while reading file: " + fileName + ". Invalid charset?");
        throw new HopTransformException(e.getMessage(), e);
    } catch (ArrayIndexOutOfBoundsException e) {
        log.logError("Error determining field types for: " + fileName + ". Inconsistent delimiters?");
        throw new HopTransformException(e.getMessage(), e);
    }
}

Example 14 with HopFileException

use of org.apache.hop.core.exception.HopFileException in project hop by apache.

the class JarCache method getNativeJars.

public Set<File> getNativeJars() throws HopFileException {
    // 
    if (nativeFiles.isEmpty()) {
        try {
            Enumeration<URL> indexFiles = getClass().getClassLoader().getResources(JarCache.ANNOTATION_INDEX_LOCATION);
            while (indexFiles.hasMoreElements()) {
                URL url = indexFiles.nextElement();
                // Relative path
                String path = url.getFile();
                File file = new File(StringUtils.substringBefore(path, "!/"));
                nativeFiles.add(file);
                // 
                try (InputStream stream = url.openStream()) {
                    IndexReader reader = new IndexReader(stream);
                    Index index = reader.read();
                    indexCache.put(file, index);
                } catch (IOException e) {
                    throw new HopFileException(MessageFormat.format("Error reading annotation index from url ''{0}''", url), e);
                }
            }
        } catch (Exception e) {
            throw new HopFileException("Error finding native plugin jar", e);
        }
    }
    return nativeFiles;
}

Example 15 with HopFileException

use of org.apache.hop.core.exception.HopFileException in project hop by apache.

the class JarCache method getIndex.

public Index getIndex(File jarFile) throws HopFileException {
    // Search annotation index from cache
    // 
    Index index = indexCache.get(jarFile);
    if (index == null) {
        try (JarFile jar = new JarFile(jarFile)) {
            ZipEntry entry = jar.getEntry(ANNOTATION_INDEX_LOCATION);
            if (entry != null) {
                try (InputStream stream = jar.getInputStream(entry)) {
                    IndexReader reader = new IndexReader(stream);
                    index = reader.read();
                    indexCache.put(jarFile, index);
                }
            }
        } catch (IOException e) {
            throw new HopFileException(MessageFormat.format("Error reading annotation index from file ''{0}''", jarFile), e);
        }
        // Cache annotation index of jars
        // 
        indexCache.put(jarFile, index);
    }
    return index;
}