Examples with DatasetUrnStateStoreNameParser org.apache.gobblin.metastore.nameParser.DatasetUrnStateStoreNameParser used on opensource projects

Example 1 with DatasetUrnStateStoreNameParser

use of org.apache.gobblin.metastore.nameParser.DatasetUrnStateStoreNameParser in project incubator-gobblin by apache.

the class FsDatasetStateStore method createStateStore.

protected static DatasetStateStore<JobState.DatasetState> createStateStore(Config config, String className) {
    // Add all job configuration properties so they are picked up by Hadoop
    Configuration conf = new Configuration();
    for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
        conf.set(entry.getKey(), entry.getValue().unwrapped().toString());
    }
    try {
        String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI);
        final FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf);
        String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY);
        Integer threadPoolOfGettingDatasetState = ConfigUtils.getInt(config, ConfigurationKeys.THREADPOOL_SIZE_OF_LISTING_FS_DATASET_STATESTORE, ConfigurationKeys.DEFAULT_THREADPOOL_SIZE_OF_LISTING_FS_DATASET_STATESTORE);
        final String datasetUrnStateStoreNameParserClass = ConfigUtils.getString(config, ConfigurationKeys.DATASETURN_STATESTORE_NAME_PARSER, SimpleDatasetUrnStateStoreNameParser.class.getName());
        LoadingCache<Path, DatasetUrnStateStoreNameParser> stateStoreNameParserLoadingCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).build(new CacheLoader<Path, DatasetUrnStateStoreNameParser>() {

            @Override
            public DatasetUrnStateStoreNameParser load(Path stateStoreDirWithStoreName) throws Exception {
                return (DatasetUrnStateStoreNameParser) GobblinConstructorUtils.invokeLongestConstructor(Class.forName(datasetUrnStateStoreNameParserClass), stateStoreFs, stateStoreDirWithStoreName);
            }
        });
        return (DatasetStateStore<JobState.DatasetState>) GobblinConstructorUtils.invokeLongestConstructor(Class.forName(className), stateStoreFs, stateStoreRootDir, threadPoolOfGettingDatasetState, stateStoreNameParserLoadingCache);
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (ReflectiveOperationException e) {
        throw new RuntimeException("Failed to instantiate " + className, e);
    }
}

Example 2 with DatasetUrnStateStoreNameParser

use of org.apache.gobblin.metastore.nameParser.DatasetUrnStateStoreNameParser in project incubator-gobblin by apache.

the class FsDatasetStateStore method sanitizeDatasetStatestoreNameFromDatasetURN.

@Override
public String sanitizeDatasetStatestoreNameFromDatasetURN(String storeName, String datasetURN) throws IOException {
    if (this.stateStoreNameParserLoadingCache == null) {
        return datasetURN;
    }
    try {
        Path statestoreDirWithStoreName = new Path(this.storeRootDir, storeName);
        DatasetUrnStateStoreNameParser datasetUrnBasedStateStoreNameParser = this.stateStoreNameParserLoadingCache.get(statestoreDirWithStoreName);
        return datasetUrnBasedStateStoreNameParser.getStateStoreNameFromDatasetUrn(datasetURN);
    } catch (ExecutionException e) {
        throw new IOException("Failed to load dataset state store name parser: " + e, e);
    }
}