1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26 package fr.ifremer.isisfish.logging.io;
27
28 import java.io.File;
29 import java.io.IOException;
30 import java.util.Arrays;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.nuiton.util.StringUtil;
35 import org.junit.Assert;
36 import org.junit.Before;
37 import org.junit.Test;
38
39 import fr.ifremer.isisfish.AbstractIsisFishTest;
40 import fr.ifremer.isisfish.logging.LoggerHelper;
41
42
43
44
45
46
47
48 public class LineReaderTest extends AbstractIsisFishTest {
49
50 protected static final Log log = LogFactory.getLog(LineReaderTest.class);
51
52 protected File testDir;
53
54 final static int[] sizes = new int[]{
55 10,
56 50,
57 100,
58 1000,
59 5000,
60
61
62
63
64
65
66
67
68 };
69
70 @Before
71 public void setUp() throws Exception {
72 testDir = new File(getTestDirectory(), "bench_" + System.nanoTime());
73 testDir.mkdirs();
74 System.out.println("benchdir " + testDir);
75
76 }
77
78 protected static void createMock(File testDir, int i) throws IOException, InterruptedException {
79 LoggerHelper.MEGA_STREAM_SIZE = 6 * i;
80 LoggerHelper.filename = new File(testDir, LineReaderTest.class.getSimpleName() + "_" + 6 * i + ".log").getAbsolutePath();
81 File file = new File(testDir, LoggerHelper.filename);
82 long t0 = System.nanoTime();
83 LoggerHelper.mockLogFile();
84 long t1;
85
86 t1 = System.nanoTime();
87 log.info("create [" + LoggerHelper.MEGA_STREAM_SIZE + "] in " + StringUtil.convertTime(t1 - t0) + " [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
88 System.gc();
89 }
90
91 public void testMakeMock() throws Exception {
92
93 for (int size : sizes) {
94 createMock(testDir, size);
95 }
96 }
97
98 public void testCreateOffsets() throws Exception {
99
100 for (int size : sizes) {
101 createOffset(size);
102 }
103 }
104
105 public void testGetNbLines() throws Exception {
106
107 for (int size : sizes) {
108 getNbLines(size);
109 }
110 }
111
112 public void testScanAllLines() throws Exception {
113
114 for (int size : sizes) {
115 scanAllLines(size * 6);
116 scanAllLines(size * 6, 20);
117 scanAllLines(size * 6, 40);
118 scanAllLines(size * 6, 100);
119 }
120
121 }
122
123 public void testScanReverseAllLines() throws Exception {
124
125 for (int size : sizes) {
126 scanReverseAllLines(size * 6);
127 scanReverseAllLines(size * 6, 20);
128 scanReverseAllLines(size * 6, 40);
129 scanReverseAllLines(size * 6, 100);
130 }
131 }
132
133 public void testScanThenReverseAllLines() throws Exception {
134
135 for (int size : sizes) {
136 scanThenReverseAllLines(size * 6);
137 scanThenReverseAllLines(size * 6, 20);
138 scanThenReverseAllLines(size * 6, 40);
139 scanThenReverseAllLines(size * 6, 100);
140 }
141 }
142
143 private boolean showLog = false;
144
145 protected LineReader createReader(File f, String ext) throws IOException {
146 return new LineReader(f, new FileOffsetReader(new File(f, ext)));
147 }
148
149 protected void scanAllLines(int size) throws IOException {
150 LoggerHelper.MEGA_STREAM_SIZE = size;
151 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
152 File file = new File(LoggerHelper.filename);
153 long t0 = System.currentTimeMillis();
154 LineReader t = createReader(file, "offsets");
155 t.open();
156 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
157 for (int i = 0; i < t.getNbLines(); i++) {
158 long offset = t.getOffsetReader().getOffset(i);
159
160 String line = t.readLine(i);
161
162 }
163 for (int i = 0; i < t.getNbLines(); i++) {
164 long offset = t.getOffsetReader().getOffset(i);
165
166 String line = t.readLine(i);
167 if (showLog) {
168 log.info(offset + " - " + line);
169 }
170 }
171 t.close();
172 long t1 = System.currentTimeMillis();
173 log.info("[" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
174 }
175
176 protected void scanAllLines(int size, int length) throws IOException {
177 LoggerHelper.MEGA_STREAM_SIZE = size;
178 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
179 File file = new File(LoggerHelper.filename);
180 long t0 = System.currentTimeMillis();
181 LineReader t = createReader(file, "offsets");
182 t.open();
183 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
184 for (int i = 0; i < t.getNbLines(); i += length) {
185 long offset = t.getOffsetReader().getOffset(i);
186
187 String[] line = t.readLine(i, length);
188
189 }
190 for (int i = 0; i < t.getNbLines(); i += length) {
191 long offset = t.getOffsetReader().getOffset(i);
192
193 String[] line = t.readLine(i, length);
194 if (showLog) {
195 log.info(offset + " - " + Arrays.toString(line));
196 }
197 }
198 t.close();
199 long t1 = System.currentTimeMillis();
200 log.info("[" + length + "][" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
201 }
202
203 protected void scanReverseAllLines(int size) throws IOException {
204 LoggerHelper.MEGA_STREAM_SIZE = size;
205 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
206 File file = new File(LoggerHelper.filename);
207 long t0 = System.currentTimeMillis();
208 LineReader t = createReader(file, "offsets");
209 t.open();
210 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
211 for (int i = 0; i < t.getNbLines(); i += 10) {
212 long offset = t.getOffsetReader().getOffset(i);
213
214 String line = t.readLine(i);
215
216 }
217 for (long i = t.getNbLines() - 1; i >= 0; i -= 10) {
218 long offset = t.getOffsetReader().getOffset(i);
219
220 String line = t.readLine(i);
221 if (showLog) {
222 log.info(offset + " - " + line);
223 }
224 }
225 t.close();
226 long t1 = System.currentTimeMillis();
227 log.info("[" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
228 }
229
230 protected void scanReverseAllLines(int size, int length) throws IOException {
231 LoggerHelper.MEGA_STREAM_SIZE = size;
232 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
233 File file = new File(LoggerHelper.filename);
234 long t0 = System.currentTimeMillis();
235 LineReader t = createReader(file, "offsets");
236 t.open();
237 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
238 for (int i = 0; i < t.getNbLines(); i += length) {
239 long offset = t.getOffsetReader().getOffset(i);
240
241 String[] line = t.readLine(i, length);
242
243 }
244 for (long i = t.getNbLines() - 1; i >= 0; i -= length) {
245 long offset = t.getOffsetReader().getOffset(i);
246
247 String[] line = t.readLine(i, length);
248 if (showLog) {
249 log.info(offset + " - " + Arrays.toString(line));
250 }
251 }
252 t.close();
253 long t1 = System.currentTimeMillis();
254 log.info("[" + length + "][" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
255 }
256
257 protected void scanThenReverseAllLines(int size) throws IOException {
258 LoggerHelper.MEGA_STREAM_SIZE = size;
259 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
260 File file = new File(LoggerHelper.filename);
261 long t0 = System.currentTimeMillis();
262 LineReader t = createReader(file, "offsets");
263 t.open();
264 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
265 for (int i = 0; i < t.getNbLines(); i += 10) {
266 long offset = t.getOffsetReader().getOffset(i);
267
268 String line = t.readLine(i);
269
270 }
271 for (long i = t.getNbLines() - 1; i >= 0; i -= 10) {
272 long offset = t.getOffsetReader().getOffset(i);
273
274 String line = t.readLine(i);
275
276 }
277 t.close();
278 long t1 = System.currentTimeMillis();
279 log.info("[" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
280 }
281
282 protected void scanThenReverseAllLines(int size, int length) throws IOException {
283 LoggerHelper.MEGA_STREAM_SIZE = size;
284 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + LoggerHelper.MEGA_STREAM_SIZE + ".log";
285 File file = new File(LoggerHelper.filename);
286 long t0 = System.currentTimeMillis();
287 LineReader t = createReader(file, "offsets");
288 t.open();
289 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
290 for (int i = 0; i < t.getNbLines(); i += length) {
291 long offset = t.getOffsetReader().getOffset(i);
292
293 String[] line = t.readLine(i, length);
294
295 }
296 for (long i = t.getNbLines() - 1; i >= 0; i -= length) {
297 long offset = t.getOffsetReader().getOffset(i);
298
299 String[] line = t.readLine(i, length);
300
301 }
302 t.close();
303 long t1 = System.currentTimeMillis();
304 log.info("[" + length + "][" + time(t0, t1) + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
305 }
306
307 protected void getNbLines(int size) throws IOException, InterruptedException {
308 LoggerHelper.MEGA_STREAM_SIZE = 6 * size;
309 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + 6 * size + ".log";
310 File file = new File(LoggerHelper.filename);
311 long t0 = System.currentTimeMillis();
312 LineReader t = createReader(file, "offsets");
313 t.open();
314 long nbLines = t.getNbLines();
315 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, nbLines);
316 t.close();
317 long t1 = System.currentTimeMillis();
318 log.info("(" + nbLines + ") [" + LoggerHelper.MEGA_STREAM_SIZE + "] in " + time(t0, t1) + " [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "]");
319 }
320
321 protected void createOffset(int size) throws IOException, InterruptedException {
322 LoggerHelper.MEGA_STREAM_SIZE = 6 * size;
323 LoggerHelper.filename = "bench/" + LineReaderTest.class.getSimpleName() + "_" + 6 * size + ".log";
324 File file = new File(LoggerHelper.filename);
325 File offsetfile = new File(LoggerHelper.filename + ".offsets");
326 if (offsetfile.exists()) {
327 if (!offsetfile.delete()) {
328 Assert.fail("could not delete file " + offsetfile);
329 }
330 }
331 long t0 = System.currentTimeMillis();
332 LineReader t = createReader(file, "offsets");
333 Assert.assertEquals(LoggerHelper.MEGA_STREAM_SIZE, t.getNbLines());
334 t.close();
335 long t1 = System.currentTimeMillis();
336 log.info("time (" + time(t0, t1) + ") [" + LoggerHelper.MEGA_STREAM_SIZE + "] [file: " + file + ", length: " + StringUtil.convertMemory(file.length()) + "] [offsetfile: " + offsetfile + ", length: " + StringUtil.convertMemory(offsetfile.length()) + "]");
337 }
338
339 public static String time(long t0, long t1) {
340 StringBuilder sb = new StringBuilder();
341 long milis = t1 - t0;
342 long total = milis;
343 long minutes, secondes;
344
345 if (milis > 1000) {
346 secondes = milis / 1000;
347 if (secondes > 60) {
348 minutes = secondes / 60;
349 secondes -= minutes * 60;
350 total -= (minutes * 1000 * 60);
351 sb.append(minutes).append("mn ");
352 }
353 total -= (secondes * 1000);
354 sb.append(secondes).append("s ");
355 }
356 sb.append(total).append("ms ");
357 sb.append(" ~ ").append((float) (t1 - t0) / LoggerHelper.MEGA_STREAM_SIZE);
358 return sb.toString();
359 }
360
361 @Test
362 public void testSimple() throws Exception {
363
364 String filename = "bench/LineReaderTest_30.log";
365
366
367 File f = new File(filename);
368 if (!f.exists()) {
369 return;
370 }
371 LineReader reader = createReader(f, "offsets");
372 reader.getOffsetReader().deleteOffsetFile();
373 reader.open();
374
375 long nbLines = reader.getNbLines();
376
377 log.info("found " + nbLines + " in reader " + reader);
378 for (int i = 0; i < nbLines; i++) {
379 long offset = reader.getOffsetReader().getOffset(i);
380 String line = reader.readLine(i);
381 System.out.println("line " + i + " offset:" + offset + " content:'" + line + "'");
382 }
383 for (int i = 0; i < nbLines; i += 10) {
384 long offset = reader.getOffsetReader().getOffset(i);
385 String[] line = reader.readLine(i, 10);
386 for (int i1 = 0; i1 < line.length; i1++) {
387 String s = line[i1];
388 System.out.println("line [" + i1 + "] " + i + " to " + (i + 10) + " offset:" + offset + " content:'" + s + "'");
389 }
390
391 }
392
393 System.out.println("");
394 System.out.println("");
395 LineReader reader2 = new LineReader(reader, new FileOffsetReader(new File(f, "offsets_debug_or_warn"))) {
396 public boolean match(String line) {
397 return line.contains("debug") || line.contains("warn");
398 }
399 };
400 reader2.getOffsetReader().deleteOffsetFile();
401 reader2.open();
402
403 nbLines = reader2.getNbLines();
404
405 log.info("found " + nbLines + " in reader " + reader2);
406 for (int i = 0; i < nbLines; i++) {
407 long offset = reader2.getOffsetReader().getOffset(i);
408 String line = reader2.readLine(i);
409 System.out.println("line " + i + " offset:" + offset + " content:'" + line + "'");
410 }
411 for (int i = 0; i < nbLines; i += 10) {
412 long offset = reader2.getOffsetReader().getOffset(i);
413 String[] line = reader2.readLine(i, 10);
414 for (int i1 = 0; i1 < line.length; i1++) {
415 String s = line[i1];
416 System.out.println("line [" + i1 + "] " + i + " to " + (i + 10) + " offset:" + offset + " content:'" + s + "'");
417 }
418 }
419
420 System.out.println("");
421 System.out.println("");
422 LineReader reader3 = new LineReader(reader, new FileOffsetReader(new File(f, "offsets_debug"))) {
423 public boolean match(String line) {
424 return line.contains("debug");
425 }
426 };
427 reader3.getOffsetReader().deleteOffsetFile();
428 reader3.open();
429
430 nbLines = reader3.getNbLines();
431
432 log.info("found " + nbLines + " in reader " + reader3);
433 for (int i = 0; i < nbLines; i++) {
434 long offset = reader3.getOffsetReader().getOffset(i);
435 String line = reader3.readLine(i);
436 System.out.println("line " + i + " offset:" + offset + " content:'" + line + "'");
437 }
438 for (int i = 0; i < nbLines; i += 10) {
439 long offset = reader3.getOffsetReader().getOffset(i);
440 String[] line = reader3.readLine(i, 10);
441 for (int i1 = 0; i1 < line.length; i1++) {
442 String s = line[i1];
443 System.out.println("line [" + i1 + "] " + i + " to " + (i + 10) + " offset:" + offset + " content:'" + s + "'");
444 }
445 }
446 reader.getOffsetReader().deleteOffsetFile();
447 reader2.getOffsetReader().deleteOffsetFile();
448 reader3.getOffsetReader().deleteOffsetFile();
449 reader.close();
450 reader2.close();
451 reader3.close();
452 }
453 }