Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 81 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class ParquetDataWriterBuilder method getWriter.

/**
 * Build a {@link ParquetWriter<Group>} for given file path with a block size.
 * @param blockSize
 * @param stagingFile
 * @return
 * @throws IOException
 */
public ParquetWriter<Group> getWriter(int blockSize, Path stagingFile) throws IOException {
    State state = this.destination.getProperties();
    int pageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_PAGE_SIZE), DEFAULT_PAGE_SIZE);
    int dictPageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_DICTIONARY_PAGE_SIZE), DEFAULT_BLOCK_SIZE);
    boolean enableDictionary = state.getPropAsBoolean(getProperty(WRITER_PARQUET_DICTIONARY), DEFAULT_IS_DICTIONARY_ENABLED);
    boolean validate = state.getPropAsBoolean(getProperty(WRITER_PARQUET_VALIDATE), DEFAULT_IS_VALIDATING_ENABLED);
    String rootURI = state.getProp(WRITER_FILE_SYSTEM_URI, LOCAL_FS_URI);
    Path absoluteStagingFile = new Path(rootURI, stagingFile);
    CompressionCodecName codec = getCodecFromConfig();
    GroupWriteSupport support = new GroupWriteSupport();
    Configuration conf = new Configuration();
    GroupWriteSupport.setSchema(this.schema, conf);
    ParquetProperties.WriterVersion writerVersion = getWriterVersion();
    return new ParquetWriter<>(absoluteStagingFile, support, codec, blockSize, pageSize, dictPageSize, enableDictionary, validate, writerVersion, conf);
}

Example 82 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class ParquetHdfsDataWriterTest method createStateWithConfig.

private State createStateWithConfig() {
    State properties = new State();
    properties.setProp(ConfigurationKeys.WRITER_BUFFER_SIZE, ConfigurationKeys.DEFAULT_BUFFER_SIZE);
    properties.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI);
    properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, TestConstants.TEST_STAGING_DIR);
    properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, TestConstants.TEST_OUTPUT_DIR);
    properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, this.filePath);
    properties.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestConstants.PARQUET_TEST_FILENAME);
    properties.setProp(WRITER_PARQUET_DICTIONARY, true);
    properties.setProp(WRITER_PARQUET_DICTIONARY_PAGE_SIZE, 1024);
    properties.setProp(WRITER_PARQUET_PAGE_SIZE, 1024);
    properties.setProp(WRITER_PARQUET_VALIDATE, true);
    return properties;
}

Example 83 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class ServiceBasedAppLauncher method addServicesFromProperties.

private void addServicesFromProperties(Properties properties) throws IllegalAccessException, InstantiationException, ClassNotFoundException, InvocationTargetException {
    if (properties.containsKey(APP_ADDITIONAL_SERVICES)) {
        for (String serviceClassName : new State(properties).getPropAsSet(APP_ADDITIONAL_SERVICES)) {
            Class<?> serviceClass = Class.forName(serviceClassName);
            if (Service.class.isAssignableFrom(serviceClass)) {
                Service service;
                Constructor<?> constructor = ConstructorUtils.getMatchingAccessibleConstructor(serviceClass, Properties.class);
                if (constructor != null) {
                    service = (Service) constructor.newInstance(properties);
                } else {
                    service = (Service) serviceClass.newInstance();
                }
                addService(service);
            } else {
                throw new IllegalArgumentException(String.format("Class %s specified by %s does not implement %s", serviceClassName, APP_ADDITIONAL_SERVICES, Service.class.getSimpleName()));
            }
        }
    }
}

Example 84 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class TaskTest method createTestsForDifferentExecutionModes.

@DataProvider(name = "stateOverrides")
public Object[][] createTestsForDifferentExecutionModes() {
    State synchronousStateOverrides = new State();
    synchronousStateOverrides.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, true);
    State streamStateOverrides = new State();
    streamStateOverrides.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, false);
    return new Object[][] { { synchronousStateOverrides }, { streamStateOverrides } };
}

Example 85 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class FsCommitSequenceStoreTest method setUp.

@BeforeClass
public void setUp() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test"));
    State props = new State();
    props.setId("propsId");
    props.setProp("prop1", "valueOfProp1");
    props.setProp("prop2", "valueOfProp2");
    DatasetState datasetState = new DatasetState();
    datasetState.setDatasetUrn(this.datasetUrn);
    datasetState.incrementJobFailures();
    this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn").beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh")).withProps(props).endStep().beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn(this.datasetUrn).withDatasetState(datasetState).withProps(props).endStep().build();
}