Examples with GobblinScopeTypes org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes used on opensource projects

Example 16 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class GobblinBrokerCreationTest method testFailIfSubBrokerAtHigherScope.

@Test
public void testFailIfSubBrokerAtHigherScope() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build();
    try {
        jobBroker.newSubscopedBuilder(new GobblinScopeInstance(GobblinScopeTypes.INSTANCE, "instance")).build();
        Assert.fail();
    } catch (IllegalArgumentException iae) {
    // expected
    }
}

Example 17 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class GobblinBrokerCreationTest method testFailIfIntermediateScopeHasNoDefault.

@Test
public void testFailIfIntermediateScopeHasNoDefault() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    // should trow error if an intermediate scope does not have a default
    try {
        topBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).build();
        Assert.fail();
    } catch (IllegalArgumentException iae) {
    // expected
    }
}

Example 18 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class KeyedScopedConfigViewImplTest method test.

@Test
public void test() {
    String key = "myKey";
    // Correct creation behavior
    Config config = ConfigFactory.parseMap(ImmutableMap.<String, String>builder().put("key1", "value1").put("key2", "value2").put(JOINER.join(key, "key2"), "value2key").put(JOINER.join(GobblinScopeTypes.JOB.name(), "key2"), "value2scope").put(JOINER.join(GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey").build());
    KeyedScopedConfigViewImpl<GobblinScopeTypes, TestResourceKey> configView = new KeyedScopedConfigViewImpl<>(GobblinScopeTypes.JOB, new TestResourceKey(key), TestFactory.NAME, config);
    Assert.assertEquals(configView.getScope(), GobblinScopeTypes.JOB);
    Assert.assertEquals(configView.getKey().toConfigurationKey(), key);
    Assert.assertEquals(configView.getKeyedConfig().getString("key2"), "value2key");
    Assert.assertEquals(configView.getScopedConfig().getString("key2"), "value2scope");
    Assert.assertEquals(configView.getKeyedScopedConfig().getString("key2"), "value2scopekey");
    Assert.assertEquals(configView.getFactorySpecificConfig().getString("key1"), "value1");
    Assert.assertEquals(configView.getFactorySpecificConfig().getString("key2"), "value2");
    Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey");
    Assert.assertEquals(configView.getConfig().getString("key1"), "value1");
}

Example 19 with GobblinScopeTypes

use of org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes in project incubator-gobblin by apache.

the class FileAwareInputStreamDataWriter method writeImpl.

/**
 * Write the contents of input stream into staging path.
 *
 * <p>
 *   WriteAt indicates the path where the contents of the input stream should be written. When this method is called,
 *   the path writeAt.getParent() will exist already, but the path writeAt will not exist. When this method is returned,
 *   the path writeAt must exist. Any data written to any location other than writeAt or a descendant of writeAt
 *   will be ignored.
 * </p>
 *
 * @param inputStream {@link FSDataInputStream} whose contents should be written to staging path.
 * @param writeAt {@link Path} at which contents should be written.
 * @param copyableFile {@link org.apache.gobblin.data.management.copy.CopyEntity} that generated this copy operation.
 * @throws IOException
 */
protected void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile) throws IOException {
    final short replication = copyableFile.getPreserve().preserve(PreserveAttributes.Option.REPLICATION) ? copyableFile.getOrigin().getReplication() : this.fs.getDefaultReplication(writeAt);
    final long blockSize = copyableFile.getPreserve().preserve(PreserveAttributes.Option.BLOCK_SIZE) ? copyableFile.getOrigin().getBlockSize() : this.fs.getDefaultBlockSize(writeAt);
    Predicate<FileStatus> fileStatusAttributesFilter = new Predicate<FileStatus>() {

        @Override
        public boolean apply(FileStatus input) {
            return input.getReplication() == replication && input.getBlockSize() == blockSize;
        }
    };
    Optional<FileStatus> persistedFile = this.recoveryHelper.findPersistedFile(this.state, copyableFile, fileStatusAttributesFilter);
    if (persistedFile.isPresent()) {
        log.info(String.format("Recovering persisted file %s to %s.", persistedFile.get().getPath(), writeAt));
        this.fs.rename(persistedFile.get().getPath(), writeAt);
    } else {
        // Copy empty directories
        if (copyableFile.getFileStatus().isDirectory()) {
            this.fs.mkdirs(writeAt);
            return;
        }
        OutputStream os = this.fs.create(writeAt, true, this.fs.getConf().getInt("io.file.buffer.size", 4096), replication, blockSize);
        if (encryptionConfig != null) {
            os = EncryptionFactory.buildStreamCryptoProvider(encryptionConfig).encodeOutputStream(os);
        }
        try {
            FileSystem defaultFS = FileSystem.get(new Configuration());
            StreamThrottler<GobblinScopeTypes> throttler = this.taskBroker.getSharedResource(new StreamThrottler.Factory<GobblinScopeTypes>(), new EmptyKey());
            ThrottledInputStream throttledInputStream = throttler.throttleInputStream().inputStream(inputStream).sourceURI(copyableFile.getOrigin().getPath().makeQualified(defaultFS.getUri(), defaultFS.getWorkingDirectory()).toUri()).targetURI(this.fs.makeQualified(writeAt).toUri()).build();
            StreamCopier copier = new StreamCopier(throttledInputStream, os).withBufferSize(this.bufferSize);
            log.info("File {}: Starting copy", copyableFile.getOrigin().getPath());
            if (isInstrumentationEnabled()) {
                copier.withCopySpeedMeter(this.copySpeedMeter);
            }
            long numBytes = copier.copy();
            long fileSize = copyableFile.getFileStatus().getLen();
            if (this.checkFileSize && numBytes != fileSize) {
                throw new IOException(String.format("Number of bytes copied doesn't match filesize for file %s.", copyableFile.getOrigin().getPath()));
            }
            this.bytesWritten.addAndGet(numBytes);
            if (isInstrumentationEnabled()) {
                log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate());
            } else {
                log.info("File {} copied.", copyableFile.getOrigin().getPath());
            }
        } catch (NotConfiguredException nce) {
            log.warn("Broker error. Some features of stream copier may not be available.", nce);
        } finally {
            os.close();
            inputStream.close();
        }
    }
}