Examples with SettableBlobMeta backtype.storm.generated.SettableBlobMeta used on opensource projects

Example 1 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class ServiceHandler method setupStormCode.

/**
     * create local topology files in blobstore and sync metadata to zk
     */
private void setupStormCode(Map<Object, Object> conf, String topologyId, String tmpJarLocation, Map<Object, Object> stormConf, StormTopology topology) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    String codeKey = StormConfig.master_stormcode_key(topologyId);
    String confKey = StormConfig.master_stormconf_key(topologyId);
    // in local mode there is no jar
    if (tmpJarLocation != null) {
        setupJar(tmpJarLocation, topologyId, blobStore, clusterState, nimbusInfo, false);
    }
    blobStore.createBlob(confKey, Utils.serialize(stormConf), new SettableBlobMeta());
    blobStore.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta());
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(confKey, nimbusInfo, BlobStoreUtils.getVersionForKey(confKey, nimbusInfo, conf));
        clusterState.setup_blobstore(codeKey, nimbusInfo, BlobStoreUtils.getVersionForKey(codeKey, nimbusInfo, conf));
    }
    LOG.info("Successfully create blobstore for topology {}", topologyId);
}

Example 2 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class ServiceHandler method setupJar.

public void setupJar(String tmpJarLocation, String topologyId, BlobStore blobStore, StormClusterState clusterState, NimbusInfo nimbusInfo, boolean update) throws Exception {
    // setup lib files
    boolean existLibs = true;
    String srcLibPath = StormConfig.stormlib_path(tmpJarLocation);
    File srcFile = new File(srcLibPath);
    if (srcFile.exists()) {
        File[] libJars = srcFile.listFiles();
        if (libJars != null) {
            for (File jar : libJars) {
                if (jar.isFile()) {
                    String libJarKey = StormConfig.master_stormlib_key(topologyId, jar.getName());
                    if (update) {
                        BlobStoreUtils.updateBlob(blobStore, libJarKey, new FileInputStream(jar));
                    } else {
                        blobStore.createBlob(libJarKey, new FileInputStream(jar), new SettableBlobMeta());
                    }
                    if (blobStore instanceof LocalFsBlobStore) {
                        clusterState.setup_blobstore(libJarKey, nimbusInfo, BlobStoreUtils.getVersionForKey(libJarKey, nimbusInfo, conf));
                    }
                }
            }
        }
    } else {
        LOG.info("No lib jars " + srcLibPath);
        existLibs = false;
    }
    // find storm jar path
    String jarPath = null;
    List<String> files = PathUtils.read_dir_contents(tmpJarLocation);
    for (String file : files) {
        if (file.endsWith(".jar")) {
            jarPath = tmpJarLocation + PathUtils.SEPERATOR + file;
            break;
        }
    }
    if (jarPath == null) {
        if (!existLibs) {
            throw new IllegalArgumentException("No jar under " + tmpJarLocation);
        } else {
            LOG.info("No submit jar");
            return;
        }
    }
    // setup storm jar
    String jarKey = StormConfig.master_stormjar_key(topologyId);
    if (update) {
        BlobStoreUtils.updateBlob(blobStore, jarKey, new FileInputStream(jarPath));
    } else {
        blobStore.createBlob(jarKey, new FileInputStream(jarPath), new SettableBlobMeta());
    }
    if (blobStore instanceof LocalFsBlobStore) {
        int versionInfo = BlobStoreUtils.getVersionForKey(jarKey, nimbusInfo, conf);
        clusterState.setup_blobstore(jarKey, nimbusInfo, versionInfo);
    }
    PathUtils.rmr(tmpJarLocation);
}

Example 3 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class blobstore method setupStormCode.

/**
 * create local topology files in blobstore and sync metadata to zk
 */
private static void setupStormCode(Map conf, String topologyId, String topologyRoot, Set<String> activeKeys) throws Exception {
    Map<String, String> blobKeysToLocation = getBlobKeysToLocation(topologyRoot, topologyId);
    for (Map.Entry<String, String> entry : blobKeysToLocation.entrySet()) {
        String key = entry.getKey();
        String location = entry.getValue();
        if (!activeKeys.contains(key)) {
            blobStore.createBlob(key, new FileInputStream(location), new SettableBlobMeta());
            if (isLocalBlobStore) {
                int versionInfo = BlobStoreUtils.getVersionForKey(key, nimbusInfo, conf);
                clusterState.setup_blobstore(key, nimbusInfo, versionInfo);
            }
        }
    }
    System.out.println("Successfully create blobstore for topology " + topologyId);
}

Example 4 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class LocalFsBlobStore method getBlobMeta.

@Override
public ReadableBlobMeta getBlobMeta(String key) throws KeyNotFoundException {
    validateKey(key);
    if (!checkForBlobOrDownload(key)) {
        checkForBlobUpdate(key);
    }
    SettableBlobMeta meta = getStoredBlobMeta(key);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(meta);
    try {
        LocalFsBlobStoreFile pf = fbs.read(DATA_PREFIX + key);
        rbm.set_version(pf.getModTime());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return rbm;
}

Example 5 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class ServiceHandler method createOrUpdateBlob.

private void createOrUpdateBlob(String key, byte[] blobData, boolean update, String topologyId) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    if (update) {
        BlobStoreUtils.updateBlob(blobStore, key, blobData);
        LOG.info("Successfully updated blobstore for topology:{}, key:{}", topologyId, key);
    } else {
        blobStore.createBlob(key, blobData, new SettableBlobMeta());
        LOG.info("Successfully created blobstore for topology:{}, key:{}", topologyId, key);
    }
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(key, nimbusInfo, BlobStoreUtils.getVersionForKey(key, nimbusInfo, conf));
    }
}