Examples with NonSyncByteArrayOutputStream org.apache.tez.common.io.NonSyncByteArrayOutputStream used on opensource projects

Example 1 with NonSyncByteArrayOutputStream

use of org.apache.tez.common.io.NonSyncByteArrayOutputStream in project tez by apache.

the class VertexImpl method assignVertexManager.

private void assignVertexManager() throws TezException {
    // VertexReconfigureDoneEvent will be logged
    if (recoveryData != null && recoveryData.shouldSkipInit()) {
        // Replace the original VertexManager with NoOpVertexManager if the reconfiguration is done in the last AM attempt
        VertexConfigurationDoneEvent reconfigureDoneEvent = recoveryData.getVertexConfigurationDoneEvent();
        if (LOG.isInfoEnabled()) {
            LOG.info("VertexManager reconfiguration is done in the last AM Attempt" + ", use NoOpVertexManager to replace it, vertexId=" + logIdentifier);
            LOG.info("VertexReconfigureDoneEvent=" + reconfigureDoneEvent);
        }
        NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream();
        try {
            reconfigureDoneEvent.toProtoStream(out);
        } catch (IOException e) {
            throw new TezUncheckedException("Unable to deserilize VertexReconfigureDoneEvent");
        }
        this.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(NoOpVertexManager.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), dagUgi, this, appContext, stateChangeNotifier);
        return;
    }
    boolean hasBipartite = false;
    boolean hasOneToOne = false;
    boolean hasCustom = false;
    if (sourceVertices != null) {
        for (Edge edge : sourceVertices.values()) {
            switch(edge.getEdgeProperty().getDataMovementType()) {
                case SCATTER_GATHER:
                    hasBipartite = true;
                    break;
                case ONE_TO_ONE:
                    hasOneToOne = true;
                    break;
                case BROADCAST:
                    break;
                case CUSTOM:
                    hasCustom = true;
                    break;
                default:
                    throw new TezUncheckedException("Unknown data movement type: " + edge.getEdgeProperty().getDataMovementType());
            }
        }
    }
    boolean hasUserVertexManager = vertexPlan.hasVertexManagerPlugin();
    if (hasUserVertexManager) {
        VertexManagerPluginDescriptor pluginDesc = DagTypeConverters.convertVertexManagerPluginDescriptorFromDAGPlan(vertexPlan.getVertexManagerPlugin());
        LOG.info("Setting user vertex manager plugin: " + pluginDesc.getClassName() + " on vertex: " + getLogIdentifier());
        vertexManager = new VertexManager(pluginDesc, dagUgi, this, appContext, stateChangeNotifier);
    } else {
        // Else we use the default ImmediateStartVertexManager
        if (inputsWithInitializers != null) {
            LOG.info("Setting vertexManager to RootInputVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(RootInputVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasOneToOne && !hasCustom) {
            LOG.info("Setting vertexManager to InputReadyVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasBipartite && !hasCustom) {
            LOG.info("Setting vertexManager to ShuffleVertexManager for " + logIdentifier);
            // shuffle vertex manager needs a conf payload
            vertexManager = new VertexManager(ShuffleVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else {
            // schedule all tasks upon vertex start. Default behavior.
            LOG.info("Setting vertexManager to ImmediateStartVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        }
    }
}

Example 2 with NonSyncByteArrayOutputStream

use of org.apache.tez.common.io.NonSyncByteArrayOutputStream in project tez by apache.

the class TezUtilsInternal method compressBytesInflateDeflate.

private static byte[] compressBytesInflateDeflate(byte[] inBytes) {
    Deflater deflater = new Deflater(Deflater.BEST_SPEED);
    deflater.setInput(inBytes);
    NonSyncByteArrayOutputStream bos = new NonSyncByteArrayOutputStream(inBytes.length);
    deflater.finish();
    byte[] buffer = new byte[1024 * 8];
    while (!deflater.finished()) {
        int count = deflater.deflate(buffer);
        bos.write(buffer, 0, count);
    }
    byte[] output = bos.toByteArray();
    return output;
}

Example 3 with NonSyncByteArrayOutputStream

use of org.apache.tez.common.io.NonSyncByteArrayOutputStream in project tez by apache.

the class TezUtilsInternal method uncompressBytesInflateDeflate.

private static byte[] uncompressBytesInflateDeflate(byte[] inBytes) throws IOException {
    Inflater inflater = new Inflater();
    inflater.setInput(inBytes);
    NonSyncByteArrayOutputStream bos = new NonSyncByteArrayOutputStream(inBytes.length);
    byte[] buffer = new byte[1024 * 8];
    while (!inflater.finished()) {
        int count;
        try {
            count = inflater.inflate(buffer);
        } catch (DataFormatException e) {
            throw new IOException(e);
        }
        bos.write(buffer, 0, count);
    }
    byte[] output = bos.toByteArray();
    return output;
}

Example 4 with NonSyncByteArrayOutputStream

use of org.apache.tez.common.io.NonSyncByteArrayOutputStream in project tez by apache.

the class ATSFileParser method readJson.

private JSONObject readJson(InputStream in) throws IOException, JSONException {
    // Read entire content to memory
    final NonSyncByteArrayOutputStream bout = new NonSyncByteArrayOutputStream();
    IOUtils.copy(in, bout);
    return new JSONObject(new String(bout.toByteArray(), "UTF-8"));
}