Examples with Location org.apache.twill.filesystem.Location used on opensource projects

Example 6 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class AuthorizationBootstrapperTest method createAppJar.

private static void createAppJar(Class<?> cls, File destFile, Manifest manifest) throws IOException {
    Location deploymentJar = AppJarHelper.createDeploymentJar(new LocalLocationFactory(TMP_FOLDER.newFolder()), cls, manifest);
    DirUtils.mkdirs(destFile.getParentFile());
    Files.copy(Locations.newInputSupplier(deploymentJar), destFile);
}

Example 7 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileStreamAdmin method updateConfig.

@Override
public void updateConfig(final StreamId streamId, final StreamProperties properties) throws Exception {
    Location streamLocation;
    streamLocation = impersonator.doAs(streamId.getParent(), new Callable<Location>() {

        @Override
        public Location call() throws Exception {
            return getStreamLocation(streamId);
        }
    });
    Preconditions.checkArgument(streamLocation.isDirectory(), "Stream '%s' does not exist.", streamId);
    SecurityUtil.verifyOwnerPrincipal(streamId, properties.getOwnerPrincipal(), ownerAdmin);
    streamCoordinatorClient.updateProperties(streamId, new Callable<CoordinatorStreamProperties>() {

        @Override
        public CoordinatorStreamProperties call() throws Exception {
            StreamProperties oldProperties = updateProperties(streamId, properties);
            FormatSpecification format = properties.getFormat();
            if (format != null) {
                // if the schema has changed, we need to recreate the hive table.
                // Changes in format and settings don't require
                // a hive change, as they are just properties used by the stream storage handler.
                Schema currSchema = oldProperties.getFormat().getSchema();
                Schema newSchema = format.getSchema();
                if (!Objects.equals(currSchema, newSchema)) {
                    alterExploreStream(streamId, false, null);
                    alterExploreStream(streamId, true, format);
                }
            }
            publishAudit(streamId, AuditType.UPDATE);
            return new CoordinatorStreamProperties(properties.getTTL(), properties.getFormat(), properties.getNotificationThresholdMB(), null, properties.getDescription(), properties.getOwnerPrincipal());
        }
    });
}

Example 8 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileStreamAdmin method writeConfig.

private void writeConfig(StreamConfig config) throws IOException {
    Location configLocation = config.getLocation().append(CONFIG_FILE_NAME);
    Location tmpConfigLocation = configLocation.getTempFile(null);
    CharStreams.write(GSON.toJson(config), CharStreams.newWriterSupplier(Locations.newOutputSupplier(tmpConfigLocation), Charsets.UTF_8));
    try {
        // Windows does not allow renaming if the destination file exists so we must delete the configLocation
        if (OSDetector.isWindows()) {
            configLocation.delete();
        }
        tmpConfigLocation.renameTo(getConfigLocation(config.getStreamId()));
    } finally {
        Locations.deleteQuietly(tmpConfigLocation);
    }
}

Example 9 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileStreamAdmin method mutateStates.

private void mutateStates(long groupId, int instances, Set<StreamConsumerState> states, Set<StreamConsumerState> newStates, Set<StreamConsumerState> removeStates) {
    int oldInstances = states.size();
    if (oldInstances == instances) {
        // If number of instances doesn't changed, no need to mutate any states
        return;
    }
    // Collects smallest offsets across all existing consumers
    // Map from event file location to file offset.
    // Use tree map to maintain ordering consistency in the offsets.
    // Not required by any logic, just easier to look at when logged.
    Map<Location, StreamFileOffset> fileOffsets = Maps.newTreeMap(Locations.LOCATION_COMPARATOR);
    for (StreamConsumerState state : states) {
        for (StreamFileOffset fileOffset : state.getState()) {
            StreamFileOffset smallestOffset = fileOffsets.get(fileOffset.getEventLocation());
            if (smallestOffset == null || fileOffset.getOffset() < smallestOffset.getOffset()) {
                fileOffsets.put(fileOffset.getEventLocation(), new StreamFileOffset(fileOffset));
            }
        }
    }
    // Constructs smallest offsets
    Collection<StreamFileOffset> smallestOffsets = fileOffsets.values();
    // When group size changed, reset all existing instances states to have smallest files offsets constructed above.
    for (StreamConsumerState state : states) {
        if (state.getInstanceId() < instances) {
            // Only keep valid instances
            newStates.add(new StreamConsumerState(groupId, state.getInstanceId(), smallestOffsets));
        } else {
            removeStates.add(state);
        }
    }
    // For all new instances, set files offsets to smallest one constructed above.
    for (int i = oldInstances; i < instances; i++) {
        newStates.add(new StreamConsumerState(groupId, i, smallestOffsets));
    }
}

Example 10 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileStreamAdmin method doDrop.

private void doDrop(final StreamId streamId, final Location streamLocation) throws Exception {
    // Delete the stream config so that calls that try to access the stream will fail after this call returns.
    // The stream coordinator client will notify all clients that stream has been deleted.
    streamCoordinatorClient.deleteStream(streamId, new Runnable() {

        @Override
        public void run() {
            try {
                final Location configLocation = impersonator.doAs(streamId, new Callable<Location>() {

                    @Override
                    public Location call() throws Exception {
                        Location configLocation = getConfigLocation(streamId);
                        return configLocation.exists() ? configLocation : null;
                    }
                });
                if (configLocation == null) {
                    return;
                }
                alterExploreStream(streamId.getParent().stream(StreamUtils.getStreamNameFromLocation(streamLocation)), false, null);
                // Drop the associated views
                List<StreamViewId> views = viewAdmin.list(streamId);
                for (StreamViewId view : views) {
                    viewAdmin.delete(view);
                }
                impersonator.doAs(streamId, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        if (!configLocation.delete()) {
                            LOG.debug("Could not delete stream config location {}", streamLocation);
                        }
                        // Move the stream directory to the deleted directory
                        // The target directory has a timestamp appended to the stream name
                        // It is for the case when a stream is created and deleted in a short period of time before
                        // the stream janitor kicks in.
                        Location deleted = StreamUtils.getDeletedLocation(getStreamBaseLocation(streamId.getParent()));
                        Locations.mkdirsIfNotExists(deleted);
                        streamLocation.renameTo(deleted.append(streamId.getEntityName() + System.currentTimeMillis()));
                        return null;
                    }
                });
                streamMetaStore.removeStream(streamId);
                ownerAdmin.delete(streamId);
                metadataStore.removeMetadata(streamId);
                publishAudit(streamId, AuditType.DELETE);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    });
}