Examples with StocatorPath com.ibm.stocator.fs.common.StocatorPath used on opensource projects

Example 1 with StocatorPath

use of com.ibm.stocator.fs.common.StocatorPath in project stocator by SparkTC.

the class StocatorPathTest method before.

@Before
public final void before() {
    mStocatorPath = PowerMockito.mock(StocatorPath.class);
    Whitebox.setInternalState(mStocatorPath, "tempIdentifiers", new String[] { pattern1 });
    Configuration conf = new Configuration();
    conf.setStrings("fs.stocator.temp.identifier", pattern1);
    stocPath = new StocatorPath(DEFAULT_FOUTPUTCOMMITTER_V1, conf, hostname);
}

Example 2 with StocatorPath

use of com.ibm.stocator.fs.common.StocatorPath in project stocator by SparkTC.

the class ObjectStoreFileSystem method initialize.

@Override
public void initialize(URI fsuri, Configuration conf) throws IOException {
    super.initialize(fsuri, conf);
    LOG.trace("Initialize for {}", fsuri);
    if (!conf.getBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", true)) {
        throw new IOException("mapreduce.fileoutputcommitter.marksuccessfuljobs should be enabled");
    }
    String escapedAuthority = UrlEscapers.urlPathSegmentEscaper().escape(fsuri.getAuthority());
    uri = URI.create(fsuri.getScheme() + "://" + escapedAuthority);
    setConf(conf);
    String committerType = conf.get(OUTPUT_COMMITTER_TYPE, DEFAULT_FOUTPUTCOMMITTER_V1);
    if (storageClient == null) {
        storageClient = ObjectStoreVisitor.getStoreClient(fsuri, conf);
        if (Utils.validSchema(fsuri.toString())) {
            hostNameScheme = storageClient.getScheme() + "://" + Utils.getHost(fsuri) + "/";
        } else {
            LOG.debug("Non valid schema for {}", fsuri.toString());
            String accessURL = Utils.extractAccessURL(fsuri.toString(), storageClient.getScheme());
            LOG.debug("Non valid schema. Access url {}", accessURL);
            String dataRoot = Utils.extractDataRoot(fsuri.toString(), accessURL);
            if (dataRoot.isEmpty()) {
                hostNameScheme = accessURL + "/";
            } else {
                hostNameScheme = accessURL + "/" + dataRoot + "/";
            }
        }
        stocatorPath = new StocatorPath(committerType, conf, hostNameScheme);
        storageClient.setStocatorPath(stocatorPath);
        storageClient.setStatistics(statistics);
    }
}