Examples with UploaderException com.twitter.heron.spi.uploader.UploaderException used on opensource projects

Example 6 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project heron by twitter.

the class LocalFileSystemUploader method uploadPackage.

/**
   * Upload the topology package to the destined location in local file system
   *
   * @return destination URI of where the topology package has
   * been uploaded if successful, or {@code null} if failed.
   */
@Override
public URI uploadPackage() throws UploaderException {
    // first, check if the topology package exists
    boolean fileExists = new File(topologyPackageLocation).isFile();
    if (!fileExists) {
        throw new UploaderException(String.format("Topology package does not exist at '%s'", topologyPackageLocation));
    }
    // get the directory containing the file
    Path filePath = Paths.get(destTopologyFile);
    File parentDirectory = filePath.getParent().toFile();
    assert parentDirectory != null;
    // if the dest directory does not exist, create it.
    if (!parentDirectory.exists()) {
        LOG.fine(String.format("Working directory does not exist. Creating it now at %s", parentDirectory.getPath()));
        if (!parentDirectory.mkdirs()) {
            throw new UploaderException(String.format("Failed to create directory for topology package at %s", parentDirectory.getPath()));
        }
    }
    // if the dest file exists, write a log message
    fileExists = new File(filePath.toString()).isFile();
    if (fileExists) {
        LOG.fine(String.format("Target topology package already exists at '%s'. Overwriting it now", filePath.toString()));
    }
    // copy the topology package to target working directory
    LOG.fine(String.format("Copying topology package at '%s' to target working directory '%s'", topologyPackageLocation, filePath.toString()));
    Path source = Paths.get(topologyPackageLocation);
    try {
        CopyOption[] options = new CopyOption[] { StandardCopyOption.REPLACE_EXISTING };
        Files.copy(source, filePath, options);
    } catch (IOException e) {
        throw new UploaderException(String.format("Unable to copy topology file from '%s' to '%s'", source, filePath), e);
    }
    return getUri(destTopologyFile);
}

Example 7 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project heron by twitter.

the class SubmitterMain method submitTopology.

/**
   * Submit a topology
   * 1. Instantiate necessary resources
   * 2. Valid whether it is legal to submit a topology
   * 3. Call LauncherRunner
   *
   */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 8 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project heron by twitter.

the class SubmitterMainTest method testSubmitTopologyUploaderException.

@Test(expected = UploaderException.class)
public void testSubmitTopologyUploaderException() throws Exception {
    SubmitterMain submitterMain = spy(new SubmitterMain(config, topology));
    doNothing().when(submitterMain).validateSubmit(any(SchedulerStateManagerAdaptor.class), anyString());
    doThrow(new UploaderException("")).when(submitterMain).uploadPackage(eq(uploader));
    try {
        submitterMain.submitTopology();
    } finally {
        verify(uploader, never()).undo();
        verify(uploader).close();
        verify(launcher).close();
        verify(statemgr).close();
    }
}

Example 9 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class S3Uploader method uploadPackage.

@Override
public URI uploadPackage() throws UploaderException {
    // Backup any existing files incase we need to undo this action
    if (s3Client.doesObjectExist(bucket, remoteFilePath)) {
        s3Client.copyObject(bucket, remoteFilePath, bucket, previousVersionFilePath);
    }
    // Attempt to write the topology package to s3
    try {
        s3Client.putObject(bucket, remoteFilePath, packageFileHandler);
    } catch (SdkClientException e) {
        throw new UploaderException(String.format("Error writing topology package to %s %s", bucket, remoteFilePath), e);
    }
    // Ask s3 for the url to the topology package we just uploaded
    final URL resourceUrl = s3Client.getUrl(bucket, remoteFilePath);
    LOG.log(Level.INFO, "Package URL: {0}", resourceUrl);
    // This will happen if the package does not actually exist in the place where we uploaded it to.
    if (resourceUrl == null) {
        throw new UploaderException(String.format("Resource not found for bucket %s and path %s", bucket, remoteFilePath));
    }
    try {
        return resourceUrl.toURI();
    } catch (URISyntaxException e) {
        throw new UploaderException(String.format("Could not convert URL %s to URI", resourceUrl), e);
    }
}

Example 10 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class GcsUploader method uploadPackage.

@Override
public URI uploadPackage() throws UploaderException {
    // Backup any existing files incase we need to undo this action
    final StorageObject previousStorageObject = gcsController.getStorageObject(topologyObjectName);
    if (previousStorageObject != null) {
        try {
            gcsController.copyStorageObject(topologyObjectName, previousTopologyObjectName, previousStorageObject);
        } catch (IOException ioe) {
            throw new UploaderException("Failed to back up previous topology", ioe);
        }
    }
    final StorageObject storageObject;
    try {
        storageObject = gcsController.createStorageObject(topologyObjectName, topologyPackageFile);
    } catch (IOException ioe) {
        throw new UploaderException(String.format("Error writing topology package to %s %s", bucket, topologyObjectName), ioe);
    }
    final String downloadUrl = getDownloadUrl(bucket, storageObject.getName());
    LOG.info("Package URL: " + downloadUrl);
    try {
        return new URI(downloadUrl);
    } catch (URISyntaxException e) {
        throw new UploaderException(String.format("Could not convert URL %s to URI", downloadUrl), e);
    }
}