Example 61 with Path
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class TextLoaderTest method importWithCommitLagAndMaxUncommittedRowsTableNotExists.
private void importWithCommitLagAndMaxUncommittedRowsTableNotExists(long commitLag, int maxUncommittedRows, boolean durable, Set<String> expectedPartitionNames) throws Exception {
final AtomicInteger rmdirCallCount = new AtomicInteger();
final AtomicInteger msyncCallCount = new AtomicInteger();
final FilesFacade ff = new TestFilesFacade() {
@Override
public int msync(long addr, long len, boolean async) {
msyncCallCount.incrementAndGet();
Assert.assertFalse(async);
return Files.msync(addr, len, false);
}
@Override
public int rmdir(Path name) {
rmdirCallCount.getAndIncrement();
if (!expectedPartitionNames.contains(extractLast(name))) {
Assert.fail();
}
return Files.rmdir(name);
}
@Override
public boolean wasCalled() {
return false;
}
};
CairoConfiguration configuration = new DefaultCairoConfiguration(root) {
@Override
public FilesFacade getFilesFacade() {
return ff;
}
@Override
public TextConfiguration getTextConfiguration() {
return new DefaultTextConfiguration() {
@Override
public int getTextAnalysisMaxLines() {
return 1;
}
};
}
@Override
public int getMaxUncommittedRows() {
return maxUncommittedRows;
}
@Override
public long getCommitLag() {
return commitLag;
}
};
try (CairoEngine engine = new CairoEngine(configuration)) {
assertNoLeak(engine, textLoader -> {
configureLoaderDefaults(textLoader, durable);
textLoader.setForceHeaders(true);
textLoader.setCommitLag(commitLag);
textLoader.setMaxUncommittedRows(maxUncommittedRows);
playText(engine, textLoader, "ts,int\n2021-01-01T00:04:00.000000Z,3\n2021-01-01T00:05:00.000000Z,4\n2021-01-02T00:05:31.000000Z,6\n2021-01-01T00:01:00.000000Z,1\n2021-01-01T00:01:30.000000Z,2\n2021-01-02T00:00:30.000000Z,5\n", 1024, "ts\tint\n" + "2021-01-01T00:01:00.000000Z\t1\n" + "2021-01-01T00:01:30.000000Z\t2\n" + "2021-01-01T00:04:00.000000Z\t3\n" + "2021-01-01T00:05:00.000000Z\t4\n" + "2021-01-02T00:00:30.000000Z\t5\n" + "2021-01-02T00:05:31.000000Z\t6\n", "{\"columnCount\":2,\"columns\":[" + "{\"index\":0,\"name\":\"ts\",\"type\":\"TIMESTAMP\"}," + "{\"index\":1,\"name\":\"int\",\"type\":\"INT\"}]," + "\"timestampIndex\":0}", 6, 6);
Assert.assertEquals("test", textLoader.getTableName());
Assert.assertEquals(TextLoadWarning.NONE, textLoader.getWarnings());
});
Assert.assertEquals(4, rmdirCallCount.get());
Assert.assertTrue((durable && msyncCallCount.get() > 0) || (!durable && msyncCallCount.get() == 0));
try (TableReader reader = engine.getReader(sqlExecutionContext.getCairoSecurityContext(), "test")) {
Assert.assertEquals(maxUncommittedRows, reader.getMaxUncommittedRows());
Assert.assertEquals(commitLag, reader.getCommitLag());
Assert.assertEquals(6, reader.size());
TestUtils.assertCursor("2021-01-01T00:01:00.000000Z 1\n2021-01-01T00:01:30.000000Z 2\n2021-01-01T00:04:00.000000Z 3\n2021-01-01T00:05:00.000000Z 4\n2021-01-02T00:00:30.000000Z 5\n2021-01-02T00:05:31.000000Z 6\n", reader.getCursor(), reader.getMetadata(), false, sink);
}
}
}
Also used :
Path(io.questdb.std.str.Path)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
Example 62 with Path
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class AlterTableAttachPartitionTest method assertSchemaMismatch.
private void assertSchemaMismatch(TableModel src, AddColumn tm) throws IOException, NumericException {
try (TableModel dst = new TableModel(configuration, "dst", PartitionBy.DAY);
Path path = new Path()) {
dst.timestamp("ts").col("i", ColumnType.INT).col("l", ColumnType.LONG);
tm.add(dst);
CairoTestUtils.create(dst);
try {
copyAttachPartition(src, dst, 0, "2020-01-10");
Assert.fail();
} catch (SqlException e) {
TestUtils.assertContains(e.getFlyweightMessage(), "Column file does not exist");
}
Files.rmdir(path.concat(root).concat("dst").concat("2020-01-10").put(TableUtils.DETACHED_DIR_MARKER).$());
}
}
Also used :
Path(io.questdb.std.str.Path)
Example 63 with Path
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class LineUdpParserImplTest method testReservedName.
@Test
public void testReservedName() throws Exception {
final String expected = "sym\tdouble\tint\tbool\tstr\ttimestamp\n" + "ok\t2.1\t11\tfalse\tdone\t2017-10-03T10:00:00.000000Z\n";
String lines = "x,sym2=xyz double=1.6,int=15i,bool=true,str=\"string1\"\n" + "x,sym1=abc double=1.3,int=11i,bool=false,str=\"string2\"\n" + "y,sym=ok double=2.1,int=11i,bool=false,str=\"done\"\n";
CairoConfiguration configuration = new DefaultCairoConfiguration(root) {
@Override
public MicrosecondClock getMicrosecondClock() {
try {
return new TestMicroClock(TimestampFormatUtils.parseTimestamp("2017-10-03T10:00:00.000Z"), 10);
} catch (NumericException e) {
throw new RuntimeException(e);
}
}
};
try (Path path = new Path()) {
Files.mkdirs(path.of(root).concat("x").slash$(), configuration.getMkDirMode());
assertThat(expected, lines, "y", configuration);
Assert.assertEquals(TableUtils.TABLE_RESERVED, TableUtils.exists(configuration.getFilesFacade(), path, root, "x"));
}
}
Also used :
Path(io.questdb.std.str.Path)
TestMicroClock(io.questdb.test.tools.TestMicroClock)
Test(org.junit.Test)
Example 64 with Path
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class IODispatcherTest method testSCPHttp10.
@Test
public void testSCPHttp10() throws Exception {
assertMemoryLeak(() -> {
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(NetworkFacadeImpl.INSTANCE, baseDir, 16 * 1024, false, false, false, "HTTP/1.0 ");
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1, -1 };
}
@Override
public int getWorkerCount() {
return 2;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
workerPool.start(LOG);
// create 20Mb file in /tmp directory
try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
try {
Rnd rnd = new Rnd();
final int diskBufferLen = 1024 * 1024;
writeRandomFile(path, rnd, 122222212222L);
long sockAddr = Net.sockaddr("127.0.0.1", 9001);
try {
int netBufferLen = 4 * 1024;
long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
try {
// send request to server to download file we just created
final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader = "HTTP/1.0 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + "Connection: close\r\n" + // this is last modified timestamp on the file, we set this value when we created file
"ETag: \"122222212222\"\r\n" + "\r\n";
for (int j = 0; j < 1; j++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
} finally {
Net.close(fd);
}
}
// send few requests to receive 304
final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
"If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader2 = "HTTP/1.0 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n";
for (int i = 0; i < 3; i++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request2, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
} finally {
Net.close(fd);
}
}
// couple more full downloads after 304
for (int j = 0; j < 2; j++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
} finally {
Net.close(fd);
}
}
// get a 404 now
final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader3 = "HTTP/1.0 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
for (int i = 0; i < 4; i++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request3, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader3, expectedResponseHeader3.length());
} finally {
Net.close(fd);
}
}
for (int i = 0; i < 3; i++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request2, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
} finally {
Net.close(fd);
}
}
} finally {
Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
}
} finally {
Net.freeSockAddr(sockAddr);
workerPool.halt();
}
} finally {
Files.remove(path);
}
}
}
});
}
Also used :
StaticContentProcessor(io.questdb.cutlass.http.processors.StaticContentProcessor)
Path(io.questdb.std.str.Path)
WorkerPool(io.questdb.mp.WorkerPool)
WorkerPoolConfiguration(io.questdb.mp.WorkerPoolConfiguration)
Example 65 with Path
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class IODispatcherTest method testSCPConnectDownloadDisconnect.
@Test
public void testSCPConnectDownloadDisconnect() throws Exception {
assertMemoryLeak(() -> {
final String baseDir = temp.getRoot().getAbsolutePath();
final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir, false);
final WorkerPool workerPool = new WorkerPool(new WorkerPoolConfiguration() {
@Override
public int[] getWorkerAffinity() {
return new int[] { -1, -1 };
}
@Override
public int getWorkerCount() {
return 2;
}
@Override
public boolean haltOnError() {
return false;
}
});
try (HttpServer httpServer = new HttpServer(httpConfiguration, workerPool, false)) {
httpServer.bind(new HttpRequestProcessorFactory() {
@Override
public HttpRequestProcessor newInstance() {
return new StaticContentProcessor(httpConfiguration);
}
@Override
public String getUrl() {
return HttpServerConfiguration.DEFAULT_PROCESSOR_URL;
}
});
workerPool.start(LOG);
// create 20Mb file in /tmp directory
try (Path path = new Path().of(baseDir).concat("questdb-temp.txt").$()) {
try {
Rnd rnd = new Rnd();
final int diskBufferLen = 1024 * 1024;
writeRandomFile(path, rnd, 122222212222L);
// httpServer.getStartedLatch().await();
long sockAddr = Net.sockaddr("127.0.0.1", 9001);
try {
int netBufferLen = 4 * 1024;
long buffer = Unsafe.calloc(netBufferLen, MemoryTag.NATIVE_DEFAULT);
try {
// send request to server to download file we just created
final String request = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader = "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + // this is last modified timestamp on the file, we set this value when we created file
"ETag: \"122222212222\"\r\n" + "\r\n";
for (int j = 0; j < 10; j++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
} finally {
Net.close(fd);
}
}
// send few requests to receive 304
final String request2 = "GET /questdb-temp.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + // this header should make static processor return 304
"If-None-Match: \"122222212222\"\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader2 = "HTTP/1.1 304 Not Modified\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "\r\n";
for (int i = 0; i < 3; i++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request2, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126);
} finally {
Net.close(fd);
}
}
// couple more full downloads after 304
for (int j = 0; j < 2; j++) {
long fd = Net.socketTcp(true);
TestUtils.assertConnect(fd, sockAddr);
try {
sendRequest(request, fd, buffer);
assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670);
} finally {
Net.close(fd);
}
}
// get a 404 now
final String request3 = "GET /questdb-temp_!.txt HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Cache-Control: max-age=0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n";
String expectedResponseHeader3 = "HTTP/1.1 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + "\r\n" + "00\r\n" + "\r\n";
sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false);
// and few more 304s
sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false);
} finally {
Unsafe.free(buffer, netBufferLen, MemoryTag.NATIVE_DEFAULT);
}
} finally {
Net.freeSockAddr(sockAddr);
}
} finally {
workerPool.halt();
Files.remove(path);
}
}
}
});
}
Also used :
StaticContentProcessor(io.questdb.cutlass.http.processors.StaticContentProcessor)
Path(io.questdb.std.str.Path)
WorkerPool(io.questdb.mp.WorkerPool)
WorkerPoolConfiguration(io.questdb.mp.WorkerPoolConfiguration)
Aggregations
Path (io.questdb.std.str.Path)141
Test (org.junit.Test)89
File (java.io.File)14
FilesFacade (io.questdb.std.FilesFacade)13
MemoryCMARW (io.questdb.cairo.vm.api.MemoryCMARW)10
MemoryMR (io.questdb.cairo.vm.api.MemoryMR)10
Rnd (io.questdb.std.Rnd)10
AbstractCairoTest (io.questdb.cairo.AbstractCairoTest)7
MemoryMA (io.questdb.cairo.vm.api.MemoryMA)7
MemoryMARW (io.questdb.cairo.vm.api.MemoryMARW)7
AtomicInteger (java.util.concurrent.atomic.AtomicInteger)7
AbstractGriffinTest (io.questdb.griffin.AbstractGriffinTest)6
NativeLPSZ (io.questdb.std.str.NativeLPSZ)6
AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6
SOCountDownLatch (io.questdb.mp.SOCountDownLatch)5
LPSZ (io.questdb.std.str.LPSZ)5
RecordCursor (io.questdb.cairo.sql.RecordCursor)4
RowCursor (io.questdb.cairo.sql.RowCursor)4
MemoryARW (io.questdb.cairo.vm.api.MemoryARW)4
RingQueue (io.questdb.mp.RingQueue)4
