Examples with HdfsPayload org.jclouds.examples.blobstore.hdfs.io.payloads.HdfsPayload used on opensource projects

Example 1 with HdfsPayload

use of org.jclouds.examples.blobstore.hdfs.io.payloads.HdfsPayload in project legacy-jclouds-examples by jclouds.

the class MainApp method upload.

/**
 * @param provider
 * @param identity
 * @param credential
 * @param hdfsUrl
 * @param containerName
 * @param objectName
 * @param plainhttp
 * @param threadcount
 * @throws IOException
 */
private void upload(String provider, String identity, String credential, String hdfsUrl, String containerName, String objectName, boolean plainhttp, String threadcount) throws IOException {
    // Init
    Properties overrides = new Properties();
    if (plainhttp)
        // default is https
        overrides.putAll(PLAIN_HTTP_ENDPOINTS);
    if (threadcount != null)
        // without setting,
        overrides.setProperty("jclouds.mpu.parallel.degree", threadcount);
    // default is 4 threads
    overrides.setProperty(provider + ".identity", identity);
    overrides.setProperty(provider + ".credential", credential);
    BlobStoreContext context = new BlobStoreContextFactory().createContext(provider, HDFS_MODULES, overrides);
    try {
        long start = System.currentTimeMillis();
        Configuration conf = getConf();
        if (conf == null) {
            conf = new Configuration();
            setConf(conf);
        }
        // Create Container
        // it can be changed to sync
        BlobStore blobStore = context.getBlobStore();
        // BlobStore
        blobStore.createContainerInLocation(null, containerName);
        Blob blob = blobStore.blobBuilder(objectName).payload(new HdfsPayload(new Path(hdfsUrl), conf)).contentType(MediaType.APPLICATION_OCTET_STREAM).contentDisposition(objectName).build();
        long length = blob.getPayload().getContentMetadata().getContentLength();
        blobStore.putBlob(containerName, blob, multipart());
        printSpeed("Sucessfully uploaded", start, length);
    } finally {
        // Close connection
        context.close();
    }
}

Example 2 with HdfsPayload

use of org.jclouds.examples.blobstore.hdfs.io.payloads.HdfsPayload in project legacy-jclouds-examples by jclouds.

the class HdfsPayloadSlicer method slice.

@Override
public Payload slice(Payload input, long offset, long length) {
    checkNotNull(input);
    checkArgument(offset >= 0, "offset is negative");
    checkArgument(length >= 0, "length is negative");
    Payload returnVal;
    if (input instanceof HdfsPayload) {
        returnVal = doSlice((FSDataInputStream) ((HdfsPayload) input).getInput(), offset, length);
        return copyMetadataAndSetLength(input, returnVal, length);
    } else {
        return super.slice(input, offset, length);
    }
}