Examples with UploaderException com.twitter.heron.spi.uploader.UploaderException used on opensource projects

Example 1 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project heron by twitter.

the class S3Uploader method uploadPackage.

@Override
public URI uploadPackage() throws UploaderException {
    // Backup any existing files incase we need to undo this action
    if (s3Client.doesObjectExist(bucket, remoteFilePath)) {
        s3Client.copyObject(bucket, remoteFilePath, bucket, previousVersionFilePath);
    }
    // Attempt to write the topology package to s3
    try {
        s3Client.putObject(bucket, remoteFilePath, packageFileHandler);
    } catch (SdkClientException e) {
        throw new UploaderException(String.format("Error writing topology package to %s %s", bucket, remoteFilePath), e);
    }
    // Ask s3 for the url to the topology package we just uploaded
    final URL resourceUrl = s3Client.getUrl(bucket, remoteFilePath);
    LOG.log(Level.INFO, "Package URL: {0}", resourceUrl);
    // This will happen if the package does not actually exist in the place where we uploaded it to.
    if (resourceUrl == null) {
        throw new UploaderException(String.format("Resource not found for bucket %s and path %s", bucket, remoteFilePath));
    }
    try {
        return resourceUrl.toURI();
    } catch (URISyntaxException e) {
        throw new UploaderException(String.format("Could not convert URL %s to URI", resourceUrl), e);
    }
}

Example 2 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class SubmitterMain method submitTopology.

/**
 * Submit a topology
 * 1. Instantiate necessary resources
 * 2. Valid whether it is legal to submit a topology
 * 3. Call LauncherRunner
 */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // initialize the uploader
        uploader.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 3 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class LocalFileSystemUploader method uploadPackage.

/**
 * Upload the topology package to the destined location in local file system
 *
 * @return destination URI of where the topology package has
 * been uploaded if successful, or {@code null} if failed.
 */
@Override
public URI uploadPackage() throws UploaderException {
    // first, check if the topology package exists
    boolean fileExists = new File(topologyPackageLocation).isFile();
    if (!fileExists) {
        throw new UploaderException(String.format("Topology package does not exist at '%s'", topologyPackageLocation));
    }
    // get the directory containing the file
    Path filePath = Paths.get(destTopologyFile);
    File parentDirectory = filePath.getParent().toFile();
    assert parentDirectory != null;
    // if the dest directory does not exist, create it.
    if (!parentDirectory.exists()) {
        LOG.fine(String.format("Working directory does not exist. Creating it now at %s", parentDirectory.getPath()));
        if (!parentDirectory.mkdirs()) {
            throw new UploaderException(String.format("Failed to create directory for topology package at %s", parentDirectory.getPath()));
        }
    }
    // if the dest file exists, write a log message
    fileExists = new File(filePath.toString()).isFile();
    if (fileExists) {
        LOG.fine(String.format("Target topology package already exists at '%s'. Overwriting it now", filePath.toString()));
    }
    // copy the topology package to target working directory
    LOG.fine(String.format("Copying topology package at '%s' to target working directory '%s'", topologyPackageLocation, filePath.toString()));
    Path source = Paths.get(topologyPackageLocation);
    try {
        CopyOption[] options = new CopyOption[] { StandardCopyOption.REPLACE_EXISTING };
        Files.copy(source, filePath, options);
    } catch (IOException e) {
        throw new UploaderException(String.format("Unable to copy topology file from '%s' to '%s'", source, filePath), e);
    }
    return getUri(destTopologyFile);
}

Example 4 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class DlogUploaderTest method testUploadPackageLocalFileNotExist.

@Test
public void testUploadPackageLocalFileNotExist() throws Exception {
    uploader = Mockito.spy(uploader);
    Namespace ns = mock(Namespace.class);
    when(nsBuilder.build()).thenReturn(ns);
    Mockito.doReturn(false).when(uploader).isLocalFileExists(Mockito.anyString());
    uploader.initialize(config);
    try {
        uploader.uploadPackage();
        fail("Should fail on uploading package");
    } catch (UploaderException ue) {
    // expected
    }
    verify(ns, never()).logExists(anyString());
}

Example 5 with UploaderException

use of com.twitter.heron.spi.uploader.UploaderException in project incubator-heron by apache.

the class DLUploader method doUploadPackage.

private URI doUploadPackage() throws IOException {
    // first, check if the topology package exists
    if (!isLocalFileExists(topologyPackageLocation)) {
        throw new UploaderException(String.format("Expected topology package file to be uploaded does not exist at '%s'", topologyPackageLocation));
    }
    // if the dest directory does not exist, create it.
    if (namespace.logExists(packageName)) {
        // if the destination file exists, write a log message
        LOG.info(String.format("Target topology file already exists at '%s'. Overwriting it now", packageURI.toString()));
        namespace.deleteLog(packageName);
    }
    // copy the topology package to target working directory
    LOG.info(String.format("Uploading topology package at '%s' to target DL at '%s'", topologyPackageLocation, packageURI.toString()));
    OutputStream out = openOutputStream(packageName);
    try {
        copier.copyFileToStream(topologyPackageLocation, out);
    } finally {
        out.close();
    }
    return packageURI;
}