Examples with FSPartFileCollection com.datatorrent.stram.util.FSPartFileCollection used on opensource projects

Example 1 with FSPartFileCollection

use of com.datatorrent.stram.util.FSPartFileCollection in project apex-core by apache.

the class FSEventRecorder method setup.

public void setup() {
    try {
        streamCodec = new JsonStreamCodec<>();
        storage = new FSPartFileCollection();
        storage.setBasePath(basePath);
        storage.setup();
        storage.writeMetaData((VERSION + "\n").getBytes());
        if (wsClient != null) {
            try {
                setupWsClient();
            } catch (ExecutionException | IOException | InterruptedException | TimeoutException ex) {
                LOG.error("Cannot connect to gateway at {}", pubSubUrl);
            }
        }
        eventRecorderThread.start();
    } catch (Exception ex) {
        throw Throwables.propagate(ex);
    }
}

Example 2 with FSPartFileCollection

use of com.datatorrent.stram.util.FSPartFileCollection in project apex-core by apache.

the class FSStatsRecorder method setup.

public void setup() {
    try {
        streamCodec = new JsonStreamCodec<>();
        containersStorage = new FSPartFileCollection();
        containersStorage.setBasePath(basePath + "/containers");
        containersStorage.setup();
        containersStorage.writeMetaData((VERSION + "\n").getBytes());
        statsRecorderThread.start();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 3 with FSPartFileCollection

use of com.datatorrent.stram.util.FSPartFileCollection in project apex-core by apache.

the class FSStatsRecorder method recordOperators.

@Override
public void recordOperators(List<OperatorInfo> operatorList, long timestamp) throws IOException {
    for (OperatorInfo operatorInfo : operatorList) {
        FSPartFileCollection operatorStorage;
        if (!logicalOperatorStorageMap.containsKey(operatorInfo.name)) {
            operatorStorage = new FSPartFileCollection();
            operatorStorage.setBasePath(basePath + "/operators/" + operatorInfo.name);
            operatorStorage.setup();
            operatorStorage.writeMetaData((VERSION + "\n").getBytes());
            logicalOperatorStorageMap.put(operatorInfo.name, operatorStorage);
        } else {
            operatorStorage = logicalOperatorStorageMap.get(operatorInfo.name);
        }
        if (!knownOperators.contains(operatorInfo.id)) {
            knownOperators.add(operatorInfo.id);
            Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "meta");
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            Slice f = streamCodec.toByteArray(fieldMap);
            bos.write(f.buffer, f.offset, f.length);
            bos.write("\n".getBytes());
            queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), true));
        }
        Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "stats");
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        Slice f = streamCodec.toByteArray(fieldMap);
        bos.write((operatorInfo.id + ":").getBytes());
        bos.write((String.valueOf(timestamp) + ":").getBytes());
        bos.write(f.buffer, f.offset, f.length);
        bos.write("\n".getBytes());
        queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), false));
    }
}