use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.
the class IgfsStreamsSelfTest method testCreateFileColocated.
/**
* @throws Exception If failed.
*/
public void testCreateFileColocated() throws Exception {
IgfsPath path = new IgfsPath("/colocated");
UUID uuid = UUID.randomUUID();
IgniteUuid affKey;
long idx = 0;
while (true) {
affKey = new IgniteUuid(uuid, idx);
if (grid(0).affinity(grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName()).mapKeyToNode(affKey).id().equals(grid(0).localNode().id()))
break;
idx++;
}
try (IgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
// Write 5M, should be enough to test distribution.
for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
}
IgfsFile info = fs.info(path);
Collection<IgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
assertEquals(1, affNodes.size());
Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
assertEquals(1, nodeIds.size());
assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}
use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.
the class IgfsStreamsSelfTest method testCreateFile.
/**
* Test file creation.
*
* @param path Path to file to store.
* @param size Size of file to store.
* @param salt Salt for file content generation.
* @throws Exception In case of any exception.
*/
private void testCreateFile(final IgfsPath path, final long size, final int salt) throws Exception {
info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']');
final AtomicInteger cnt = new AtomicInteger(0);
final Collection<IgfsPath> cleanUp = new ConcurrentLinkedQueue<>();
long time = runMultiThreaded(new Callable<Object>() {
@Override
public Object call() throws Exception {
int id = cnt.incrementAndGet();
IgfsPath f = new IgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : ""));
try (IgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) {
assertNotNull(out);
// Add all created into cleanup list.
cleanUp.add(f);
U.copy(new IgfsTestInputStream(size, salt), out);
}
return null;
}
}, WRITING_THREADS_CNT, "perform-multi-thread-writing");
if (time > 0) {
double rate = size * 1000. / time / 1024 / 1024;
info(String.format("Write file [path=%s, size=%d kB, rate=%2.1f MB/s]", path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate));
}
info("Read and validate saved file: " + path);
final InputStream expIn = new IgfsTestInputStream(size, salt);
final IgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10);
// Validate continuous reading of whole file.
assertEqualStreams(expIn, actIn, size, null);
// Validate random seek and reading.
final Random rnd = new Random();
runMultiThreaded(new Callable<Object>() {
@Override
public Object call() throws Exception {
long skip = Math.abs(rnd.nextLong() % (size + 1));
long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400));
assertEqualStreams(new IgfsTestInputStream(size, salt), actIn, range, skip);
return null;
}
}, READING_THREADS_CNT, "validate-multi-thread-reading");
expIn.close();
actIn.close();
info("Get stored file info: " + path);
IgfsFile desc = fs.info(path);
info("Validate stored file info: " + desc);
assertNotNull(desc);
if (log.isDebugEnabled())
log.debug("File descriptor: " + desc);
Collection<IgfsBlockLocation> aff = fs.affinity(path, 0, desc.length());
assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty());
int blockSize = desc.blockSize();
assertEquals("File size", size, desc.length());
assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize);
// assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString());
// assertEquals("Permission sticky bit marks this is file", false, desc.getPermission().getStickyBit());
assertEquals("Type", true, desc.isFile());
assertEquals("Type", false, desc.isDirectory());
info("Cleanup files: " + cleanUp);
for (IgfsPath f : cleanUp) {
fs.delete(f, true);
assertNull(fs.info(f));
}
}
use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.
the class IgniteHadoopFileSystem method listStatus.
/**
* {@inheritDoc}
*/
@Override
public FileStatus[] listStatus(Path f) throws IOException {
A.notNull(f, "f");
enterBusy();
try {
IgfsPath path = convert(f);
Collection<IgfsFile> list = rmtClient.listFiles(path);
if (list == null)
throw new FileNotFoundException("File " + f + " does not exist.");
List<IgfsFile> files = new ArrayList<>(list);
FileStatus[] arr = new FileStatus[files.size()];
for (int i = 0; i < arr.length; i++) arr[i] = convert(files.get(i));
if (clientLog.isLogEnabled()) {
String[] fileArr = new String[arr.length];
for (int i = 0; i < arr.length; i++) fileArr[i] = arr[i].getPath().toString();
clientLog.logListDirectory(path, fileArr);
}
return arr;
} finally {
leaveBusy();
}
}
use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.
the class IgfsProcessorSelfTest method testMakeListDeleteDirsMultithreaded.
/**
* Test make directories in multi-threaded environment.
*
* @throws Exception In case of any exception.
*/
@SuppressWarnings("TooBroadScope")
public void testMakeListDeleteDirsMultithreaded() throws Exception {
assertListDir("/");
final int max = 2 * 1000;
final int threads = 50;
final AtomicInteger cnt = new AtomicInteger();
info("Create directories: " + max);
GridTestUtils.runMultiThreaded(new Callable<Object>() {
@Override
public Object call() throws Exception {
for (int cur = cnt.incrementAndGet(); cur < max; cur = cnt.incrementAndGet()) igfs.mkdirs(path(cur));
return null;
}
}, threads, "grid-test-make-directories");
info("Validate directories were created.");
// Reset counter.
cnt.set(0);
GridTestUtils.runMultiThreaded(new Callable<Object>() {
@Override
public Object call() throws Exception {
for (int cur = cnt.incrementAndGet(); cur < max; cur = cnt.incrementAndGet()) {
IgfsFile info = igfs.info(path(cur));
assertNotNull("Expects file exist: " + cur, info);
assertTrue("Expects file is a directory: " + cur, info.isDirectory());
}
return null;
}
}, threads, "grid-test-check-directories-exist");
info("Validate directories removing.");
// Reset counter.
cnt.set(0);
GridTestUtils.runMultiThreaded(new Callable<Object>() {
@Override
public Object call() throws Exception {
for (int cur = cnt.incrementAndGet(); cur < max; cur = cnt.incrementAndGet()) igfs.delete(path(cur), true);
return null;
}
}, threads, "grid-test-delete-directories");
}
use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.
the class IgfsProcessorSelfTest method assertListDir.
/**
* Validate directory listing.
*
* @param path Directory path to validate listing for.
* @param item List of directory items.
*/
private void assertListDir(String path, String... item) {
Collection<IgfsFile> files = igfs.listFiles(new IgfsPath(path));
List<String> names = new ArrayList<>(item.length);
for (IgfsFile file : files) names.add(file.path().name());
Arrays.sort(item);
Collections.sort(names);
assertEquals(Arrays.asList(item), names);
}
Aggregations