Examples with UnmanagedFileSplit org.apache.hyracks.api.io.UnmanagedFileSplit used on opensource projects

Example 1 with UnmanagedFileSplit

use of org.apache.hyracks.api.io.UnmanagedFileSplit in project asterixdb by apache.

the class WordCountMain method parseFileSplits.

private static FileSplit[] parseFileSplits(String fileSplits) {
    String[] splits = fileSplits.split(",");
    FileSplit[] fSplits = new FileSplit[splits.length];
    for (int i = 0; i < splits.length; ++i) {
        String s = splits[i].trim();
        int idx = s.indexOf(':');
        if (idx < 0) {
            throw new IllegalArgumentException("File split " + s + " not well formed");
        }
        File file = new File(s.substring(idx + 1));
        fSplits[i] = new UnmanagedFileSplit(s.substring(0, idx), file.getAbsolutePath());
        fileSize += file.length();
    }
    return fSplits;
}

Example 2 with UnmanagedFileSplit

use of org.apache.hyracks.api.io.UnmanagedFileSplit in project asterixdb by apache.

the class FileSplitUtils method parseFileSplits.

public static FileSplit[] parseFileSplits(String fileSplits) {
    String[] splits = fileSplits.split(",");
    FileSplit[] fSplits = new FileSplit[splits.length];
    for (int i = 0; i < splits.length; ++i) {
        String s = splits[i].trim();
        int idx = s.indexOf(':');
        if (idx < 0) {
            throw new IllegalArgumentException("File split " + s + " not well formed");
        }
        fSplits[i] = new UnmanagedFileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath());
    }
    return fSplits;
}

Example 3 with UnmanagedFileSplit

use of org.apache.hyracks.api.io.UnmanagedFileSplit in project asterixdb by apache.

the class QueryTranslator method handleWriteStatement.

protected Pair<IAWriterFactory, FileSplit> handleWriteStatement(Statement stmt) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    WriteStatement ws = (WriteStatement) stmt;
    File f = new File(ws.getFileName());
    FileSplit outputFile = new UnmanagedFileSplit(ws.getNcName().getValue(), f.getPath());
    IAWriterFactory writerFactory = null;
    if (ws.getWriterClassName() != null) {
        writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance();
    }
    return new Pair<>(writerFactory, outputFile);
}

Example 4 with UnmanagedFileSplit

use of org.apache.hyracks.api.io.UnmanagedFileSplit in project asterixdb by apache.

the class Common method parseFileSplits.

static FileSplit[] parseFileSplits(String fileSplits) {
    String[] splits = fileSplits.split(",");
    FileSplit[] fSplits = new FileSplit[splits.length];
    for (int i = 0; i < splits.length; ++i) {
        String s = splits[i].trim();
        int idx = s.indexOf(':');
        if (idx < 0) {
            throw new IllegalArgumentException("File split " + s + " not well formed");
        }
        fSplits[i] = new UnmanagedFileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath());
    }
    return fSplits;
}

Example 5 with UnmanagedFileSplit

use of org.apache.hyracks.api.io.UnmanagedFileSplit in project asterixdb by apache.

the class LocalFSInputStreamFactory method configureFileSplits.

private void configureFileSplits(ICcApplicationContext appCtx, String[] splits) throws AsterixException {
    INodeResolver resolver = getNodeResolver();
    Map<InetAddress, Set<String>> ncMap = RuntimeUtils.getForcedNodeControllerMap(appCtx);
    Set<String> ncs = ncMap.values().stream().flatMap(Collection::stream).collect(Collectors.toSet());
    inputFileSplits = new UnmanagedFileSplit[splits.length];
    String node;
    String path;
    int count = 0;
    String trimmedValue;
    for (String splitPath : splits) {
        trimmedValue = splitPath.trim();
        if (!trimmedValue.contains("://")) {
            throw new AsterixException("Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\"");
        }
        node = resolver.resolveNode(appCtx, trimmedValue.split(":")[0], ncMap, ncs);
        path = trimmedValue.split("://")[1];
        inputFileSplits[count++] = new UnmanagedFileSplit(node, path);
    }
}