UtilizationTest


1   /*-
2    * See the file LICENSE for redistribution information.
3    *
4    * Copyright (c) 2002,2006 Oracle.  All rights reserved.
5    *
6    * $Id: UtilizationTest.java,v 1.21 2006/11/17 23:47:28 mark Exp $
7    */
8   
9   package com.sleepycat.je.cleaner;
10  
11  import java.io.File  ;
12  import java.io.IOException  ;
13  import java.nio.ByteBuffer  ;
14  import java.util.Enumeration  ;
15  
16  import junit.framework.Test;
17  import junit.framework.TestCase;
18  import junit.framework.TestSuite;
19  
20  import com.sleepycat.bind.tuple.IntegerBinding;
21  import com.sleepycat.je.CheckpointConfig;
22  import com.sleepycat.je.Cursor;
23  import com.sleepycat.je.Database;
24  import com.sleepycat.je.DatabaseConfig;
25  import com.sleepycat.je.DatabaseEntry;
26  import com.sleepycat.je.DatabaseException;
27  import com.sleepycat.je.DbInternal;
28  import com.sleepycat.je.Environment;
29  import com.sleepycat.je.EnvironmentConfig;
30  import com.sleepycat.je.OperationStatus;
31  import com.sleepycat.je.Transaction;
32  import com.sleepycat.je.config.EnvironmentParams;
33  import com.sleepycat.je.log.FileHeader;
34  import com.sleepycat.je.log.FileManager;
35  import com.sleepycat.je.log.LogManager;
36  import com.sleepycat.je.log.LogSource;
37  import com.sleepycat.je.log.LogUtils;
38  import com.sleepycat.je.util.TestUtils;
39  import com.sleepycat.je.utilint.DbLsn;
40  
41  public class UtilizationTest extends TestCase {
42  
43      private static final String   DB_NAME = "foo";
44  
45      private static final String   OP_NONE = "op-none";
46      private static final String   OP_CHECKPOINT = "op-checkpoint";
47      private static final String   OP_RECOVER = "op-recover";
48      //private static final String[] OPERATIONS = { OP_NONE, };
49      //*
50      private static final String  [] OPERATIONS = { OP_NONE,
51                                                   OP_CHECKPOINT,
52                                                   OP_RECOVER,
53                                                   OP_RECOVER };
54      //*/
55  
56      /*
57       * Set fetchObsoleteSize=true only for the second OP_RECOVER test.
58       * We check that OP_RECOVER works with without fetching, but with fetching
59       * we check that all LN sizes are counted.
60       */
61      private static final boolean[] FETCH_OBSOLETE_SIZE = { false,
62                                                             false,
63                                                             false,
64                                                             true };
65  
66      private static final CheckpointConfig forceConfig = new CheckpointConfig();
67      static {
68          forceConfig.setForce(true);
69      }
70  
71      private File   envHome;
72      private Environment env;
73      private Database db;
74      private boolean dups = false;
75      private DatabaseEntry keyEntry = new DatabaseEntry();
76      private DatabaseEntry dataEntry = new DatabaseEntry();
77      private String   operation;
78      private long lastFileSeen;
79      private boolean fetchObsoleteSize;
80  
81      public static Test suite() {
82          TestSuite allTests = new TestSuite();
83          for (int i = 0; i < OPERATIONS.length; i += 1) {
84              TestSuite suite = new TestSuite(UtilizationTest.class);
85              Enumeration   e = suite.tests();
86              while (e.hasMoreElements()) {
87                  UtilizationTest test = (UtilizationTest) e.nextElement();
88                  test.init(OPERATIONS[i], FETCH_OBSOLETE_SIZE[i]);
89                  allTests.addTest(test);
90              }
91          }
92          return allTests;
93      }
94  
95      public UtilizationTest() {
96          envHome = new File  (System.getProperty(TestUtils.DEST_DIR));
97      }
98  
99      private void init(String   operation, boolean fetchObsoleteSize) {
100         this.operation = operation;
101         this.fetchObsoleteSize = fetchObsoleteSize;
102     }
103 
104     public void setUp()
105         throws IOException  , DatabaseException {
106 
107         TestUtils.removeLogFiles("Setup", envHome, false);
108         TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX);
109     }
110 
111     public void tearDown()
112         throws IOException  , DatabaseException {
113         
114         /* Set test name for reporting; cannot be done in the ctor or setUp. */
115         setName(operation +
116                 (fetchObsoleteSize ? "-fetch" : "") +
117                 ':' + getName());
118 
119         try {
120             if (env != null) {
121                 env.close();
122             }
123         } catch (Throwable   e) {
124             System.out.println("tearDown: " + e);
125         }
126                 
127         try {
128             //*
129             TestUtils.removeLogFiles("tearDown", envHome, true);
130             TestUtils.removeFiles("tearDown", envHome, FileManager.DEL_SUFFIX);
131             //*/
132         } catch (Throwable   e) {
133             System.out.println("tearDown: " + e);
134         }
135 
136         db = null;
137         env = null;
138         envHome = null;
139         keyEntry = null;
140         dataEntry = null;
141     }
142 
143     /**
144      * Opens the environment and database.
145      */
146     private void openEnv()
147         throws DatabaseException {
148 
149         EnvironmentConfig config = TestUtils.initEnvConfig();
150     DbInternal.disableParameterValidation(config);
151         config.setTransactional(true);
152         config.setTxnNoSync(true);
153         config.setAllowCreate(true);
154         /* Do not run the daemons. */
155         config.setConfigParam
156             (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
157         config.setConfigParam
158             (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
159         config.setConfigParam
160         (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false");
161         config.setConfigParam
162             (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false");
163         /* Use a tiny log file size to write one LN per file. */
164         config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(),
165                               Integer.toString(64));
166         /* Don't use NIO direct buffers or we run out of memory. */
167         config.setConfigParam
168             (EnvironmentParams.LOG_DIRECT_NIO.getName(), "false");
169         
170         /* Obsolete LN size counting is optional per test. */
171         if (fetchObsoleteSize) {
172             config.setConfigParam
173                 (EnvironmentParams.CLEANER_FETCH_OBSOLETE_SIZE.getName(),
174                  "true");
175         }
176 
177         env = new Environment(envHome, config);
178 
179         /* Speed up test that uses lots of very small files. */
180         DbInternal.envGetEnvironmentImpl(env).
181                    getFileManager().
182                    setSyncAtFileEnd(false);
183 
184         openDb();
185     }
186 
187     /**
188      * Opens the database.
189      */
190     private void openDb()
191         throws DatabaseException {
192 
193         DatabaseConfig dbConfig = new DatabaseConfig();
194         dbConfig.setTransactional(true);
195         dbConfig.setAllowCreate(true);
196         dbConfig.setSortedDuplicates(dups);
197         db = env.openDatabase(null, DB_NAME, dbConfig);
198     }
199 
200     /**
201      * Closes the environment and database.
202      */
203     private void closeEnv(boolean doCheckpoint)
204         throws DatabaseException {
205 
206         /* Verify utilization using UtilizationFileReader. */
207         CleanerTestUtils.verifyUtilization
208             (DbInternal.envGetEnvironmentImpl(env),
209              true, // expectAccurateObsoleteLNCount
210              expectAccurateObsoleteLNSize());
211 
212         if (db != null) {
213             db.close();
214             db = null;
215         }
216         if (env != null) {
217             DbInternal.envGetEnvironmentImpl(env).close(doCheckpoint);
218             env = null;
219         }
220     }
221 
222     public void testReuseSlotAfterDelete()
223         throws DatabaseException {
224 
225         openEnv();
226 
227         /* Insert and delete without compress to create a knownDeleted slot. */
228         Transaction txn = env.beginTransaction(null, null);
229         long file0 = doPut(0, txn);
230         long file1 = doDelete(0, txn);
231         txn.commit();
232 
233         /* Insert key 0 to reuse the knownDeleted slot. */
234         txn = env.beginTransaction(null, null);
235         long file2 = doPut(0, txn);
236         /* Delete and insert to reuse deleted slot in same txn. */
237         long file3 = doDelete(0, txn);
238         long file4 = doPut(0, txn);
239         txn.commit();
240         performRecoveryOperation();
241 
242         expectObsolete(file0, true);
243         expectObsolete(file1, true);
244         expectObsolete(file2, true);
245         expectObsolete(file3, true);
246         expectObsolete(file4, false);
247 
248         closeEnv(true);
249     }
250 
251     public void testReuseKnownDeletedSlot()
252         throws DatabaseException {
253 
254         openEnv();
255 
256         /* Insert key 0 and abort to create a knownDeleted slot.  */
257         Transaction txn = env.beginTransaction(null, null);
258         long file0 = doPut(0, txn);
259         txn.abort();
260 
261         /* Insert key 0 to reuse the knownDeleted slot. */
262         txn = env.beginTransaction(null, null);
263         long file1 = doPut(0, txn);
264         txn.commit();
265         performRecoveryOperation();
266 
267         /* Verify that file0 is still obsolete. */
268         expectObsolete(file0, true);
269         expectObsolete(file1, false);
270 
271         closeEnv(true);
272     }
273 
274     public void testReuseKnownDeletedSlotAbort()
275         throws DatabaseException {
276 
277         openEnv();
278 
279         /* Insert key 0 and abort to create a knownDeleted slot.  */
280         Transaction txn = env.beginTransaction(null, null);
281         long file0 = doPut(0, txn);
282         txn.abort();
283 
284         /* Insert key 0 to reuse the knownDeleted slot, and abort. */
285         txn = env.beginTransaction(null, null);
286         long file1 = doPut(0, txn);
287         txn.abort();
288         performRecoveryOperation();
289 
290         /* Verify that file0 is still obsolete. */
291         expectObsolete(file0, true);
292         expectObsolete(file1, true);
293 
294         closeEnv(true);
295     }
296 
297     public void testReuseKnownDeletedSlotDup()
298         throws DatabaseException {
299 
300         dups = true;
301         openEnv();
302 
303         /* Insert two key 0 dups and checkpoint. */
304         Transaction txn = env.beginTransaction(null, null);
305         long file0 = doPut(0, 0, txn); // 1st LN
306         long file2 = doPut(0, 1, txn); // 2nd LN
307         long file1 = file2 - 1;        // DupCountLN
308         txn.commit();
309         env.checkpoint(forceConfig);
310 
311         /* Insert {0, 2} and abort to create a knownDeleted slot. */
312         txn = env.beginTransaction(null, null);
313         long file3 = doPut(0, 2, txn); // 3rd LN
314         long file4 = file3 + 1;        // DupCountLN
315         txn.abort();
316 
317         /* Insert {0, 2} to reuse the knownDeleted slot. */
318         txn = env.beginTransaction(null, null);
319         long file5 = doPut(0, 2, txn); // 4th LN
320         long file6 = file5 + 1;        // DupCountLN
321         txn.commit();
322         performRecoveryOperation();
323 
324         /* Verify that file3 is still obsolete. */
325         expectObsolete(file0, false);
326         expectObsolete(file1, true);
327         expectObsolete(file2, false);
328         expectObsolete(file3, true);
329         expectObsolete(file4, true);
330         expectObsolete(file5, false);
331         expectObsolete(file6, false);
332 
333         closeEnv(true);
334     }
335 
336     public void testReuseKnownDeletedSlotDupAbort()
337         throws DatabaseException {
338 
339         dups = true;
340         openEnv();
341 
342         /* Insert two key 0 dups and checkpoint. */
343         Transaction txn = env.beginTransaction(null, null);
344         long file0 = doPut(0, 0, txn); // 1st LN
345         long file2 = doPut(0, 1, txn); // 2nd LN
346         long file1 = file2 - 1;        // DupCountLN
347         txn.commit();
348         env.checkpoint(forceConfig);
349 
350         /* Insert {0, 2} and abort to create a knownDeleted slot. */
351         txn = env.beginTransaction(null, null);
352         long file3 = doPut(0, 2, txn); // 3rd LN
353         long file4 = file3 + 1;        // DupCountLN
354         txn.abort();
355 
356         /* Insert {0, 2} to reuse the knownDeleted slot, then abort. */
357         txn = env.beginTransaction(null, null);
358         long file5 = doPut(0, 2, txn); // 4th LN
359         long file6 = file5 + 1;        // DupCountLN
360         txn.abort();
361         performRecoveryOperation();
362 
363         /* Verify that file3 is still obsolete. */
364         expectObsolete(file0, false);
365         expectObsolete(file1, false);
366         expectObsolete(file2, false);
367         expectObsolete(file3, true);
368         expectObsolete(file4, true);
369         expectObsolete(file5, true);
370         expectObsolete(file6, true);
371 
372         closeEnv(true);
373     }
374 
375     public void testInsert()
376         throws DatabaseException {
377 
378         openEnv();
379 
380         /* Insert key 0. */
381         long file0 = doPut(0, true);
382         performRecoveryOperation();
383 
384         /* Expect that LN is not obsolete. */
385         FileSummary summary = getSummary(file0);
386         assertEquals(1, summary.totalLNCount);
387         assertEquals(0, summary.obsoleteLNCount);
388 
389         closeEnv(true);
390     }
391 
392     public void testInsertAbort()
393         throws DatabaseException {
394 
395         openEnv();
396 
397         /* Insert key 0. */
398         long file0 = doPut(0, false);
399         performRecoveryOperation();
400 
401         /* Expect that LN is obsolete. */
402         FileSummary summary = getSummary(file0);
403         assertEquals(1, summary.totalLNCount);
404         assertEquals(1, summary.obsoleteLNCount);
405 
406         closeEnv(true);
407     }
408 
409     public void testInsertDup()
410         throws DatabaseException {
411 
412         dups = true;
413         openEnv();
414 
415         /* Insert key 0 and a dup. */
416         Transaction txn = env.beginTransaction(null, null);
417         long file0 = doPut(0, 0, txn);
418         long file3 = doPut(0, 1, txn);
419         txn.commit();
420         performRecoveryOperation();
421 
422         /*
423          * The dup tree is created on 2nd insert.  In between the two
424          * DupCountLNs are two INs.
425          */
426         long file1 = file0 + 1; // DupCountLN (provisional)
427         long file2 = file1 + 3; // DupCountLN (non-provisional)
428         assertEquals(file3, file2 + 1); // new LN
429 
430         expectObsolete(file0, false); // 1st LN
431         expectObsolete(file1, true);  // 1st DupCountLN
432         expectObsolete(file2, false); // 2nd DupCountLN
433         expectObsolete(file3, false); // 2nd LN
434 
435         closeEnv(true);
436     }
437 
438     public void testInsertDupAbort()
439         throws DatabaseException {
440 
441         dups = true;
442         openEnv();
443 
444         /* Insert key 0 and a dup. */
445         Transaction txn = env.beginTransaction(null, null);
446         long file0 = doPut(0, 0, txn);
447         long file3 = doPut(0, 1, txn);
448         txn.abort();
449         performRecoveryOperation();
450 
451         /*
452          * The dup tree is created on 2nd insert.  In between the two
453          * DupCountLNs are two INs.
454          */
455         long file1 = file0 + 1; // DupCountLN (provisional)
456         long file2 = file1 + 3; // DupCountLN (non-provisional)
457         assertEquals(file3, file2 + 1); // new LN
458 
459         expectObsolete(file0, true);  // 1st LN
460         expectObsolete(file1, false); // 1st DupCountLN
461         expectObsolete(file2, true);  // 2nd DupCountLN
462         expectObsolete(file3, true);  // 2nd LN
463 
464         closeEnv(true);
465     }
466 
467     public void testUpdate()
468         throws DatabaseException {
469 
470         openEnv();
471 
472         /* Insert key 0 and checkpoint. */
473         long file0 = doPut(0, true);
474         env.checkpoint(forceConfig);
475 
476         /* Update key 0. */
477         long file1 = doPut(0, true);
478         performRecoveryOperation();
479 
480         expectObsolete(file0, true);
481         expectObsolete(file1, false);
482 
483         closeEnv(true);
484     }
485 
486     public void testUpdateAbort()
487         throws DatabaseException {
488 
489         openEnv();
490 
491         /* Insert key 0 and checkpoint. */
492         long file0 = doPut(0, true);
493         env.checkpoint(forceConfig);
494 
495         /* Update key 0 and abort. */
496         long file1 = doPut(0, false);
497         performRecoveryOperation();
498 
499         expectObsolete(file0, false);
500         expectObsolete(file1, true);
501 
502         closeEnv(true);
503     }
504 
505     public void testUpdateDup()
506         throws DatabaseException {
507 
508         dups = true;
509         openEnv();
510 
511         /* Insert two key 0 dups and checkpoint. */
512         Transaction txn = env.beginTransaction(null, null);
513         long file0 = doPut(0, 0, txn); // 1st LN
514         long file2 = doPut(0, 1, txn); // 2nd LN
515         long file1 = file2 - 1;        // DupCountLN
516         txn.commit();
517         env.checkpoint(forceConfig);
518 
519         /* Update {0, 0}. */
520         txn = env.beginTransaction(null, null);
521         long file3 = doUpdate(0, 0, txn); // 3rd LN
522         txn.commit();
523         performRecoveryOperation();
524 
525         expectObsolete(file0, true);
526         expectObsolete(file1, false);
527         expectObsolete(file2, false);
528         expectObsolete(file3, false);
529 
530         closeEnv(true);
531     }
532 
533     public void testUpdateDupAbort()
534         throws DatabaseException {
535 
536         dups = true;
537         openEnv();
538 
539         /* Insert two key 0 dups and checkpoint. */
540         Transaction txn = env.beginTransaction(null, null);
541         long file0 = doPut(0, 0, txn); // 1st LN
542         long file2 = doPut(0, 1, txn); // 2nd LN
543         long file1 = file2 - 1;        // DupCountLN
544         txn.commit();
545         env.checkpoint(forceConfig);
546 
547         /* Update {0, 0}. */
548         txn = env.beginTransaction(null, null);
549         long file3 = doUpdate(0, 0, txn); // 3rd LN
550         txn.abort();
551         performRecoveryOperation();
552 
553         expectObsolete(file0, false);
554         expectObsolete(file1, false);
555         expectObsolete(file2, false);
556         expectObsolete(file3, true);
557 
558         closeEnv(true);
559     }
560 
561     public void testDelete()
562         throws DatabaseException {
563 
564         openEnv();
565 
566         /* Insert key 0 and checkpoint. */
567         long file0 = doPut(0, true);
568         env.checkpoint(forceConfig);
569 
570         /* Delete key 0. */
571         long file1 = doDelete(0, true);
572         performRecoveryOperation();
573 
574         expectObsolete(file0, true);
575         expectObsolete(file1, true);
576 
577         closeEnv(true);
578     }
579 
580     public void testDeleteAbort()
581         throws DatabaseException {
582 
583         openEnv();
584 
585         /* Insert key 0 and checkpoint. */
586         long file0 = doPut(0, true);
587         env.checkpoint(forceConfig);
588 
589         /* Delete key 0 and abort. */
590         long file1 = doDelete(0, false);
591         performRecoveryOperation();
592 
593         expectObsolete(file0, false);
594         expectObsolete(file1, true);
595 
596         closeEnv(true);
597     }
598 
599     public void testDeleteDup()
600         throws DatabaseException {
601 
602         dups = true;
603         openEnv();
604 
605         /* Insert two key 0 dups and checkpoint. */
606         Transaction txn = env.beginTransaction(null, null);
607         long file0 = doPut(0, 0, txn); // 1st LN
608         long file2 = doPut(0, 1, txn); // 2nd LN
609         long file1 = file2 - 1;        // DupCountLN
610         txn.commit();
611         env.checkpoint(forceConfig);
612 
613         /* Delete {0, 0} and abort. */
614         txn = env.beginTransaction(null, null);
615         long file3 = doDelete(0, 0, txn); // 3rd LN
616         long file4 = file3 + 1;           // DupCountLN
617         txn.commit();
618         performRecoveryOperation();
619 
620         expectObsolete(file0, true);
621         expectObsolete(file1, true);
622         expectObsolete(file2, false);
623         expectObsolete(file3, true);
624         expectObsolete(file4, false);
625 
626         closeEnv(true);
627     }
628 
629     public void testDeleteDupAbort()
630         throws DatabaseException {
631 
632         dups = true;
633         openEnv();
634 
635         /* Insert two key 0 dups and checkpoint. */
636         Transaction txn = env.beginTransaction(null, null);
637         long file0 = doPut(0, 0, txn); // 1st LN
638         long file2 = doPut(0, 1, txn); // 2nd LN
639         long file1 = file2 - 1;        // DupCountLN
640         txn.commit();
641         env.checkpoint(forceConfig);
642 
643         /* Delete {0, 0} and abort. */
644         txn = env.beginTransaction(null, null);
645         long file3 = doDelete(0, 0, txn); // 3rd LN
646         long file4 = file3 + 1;           // DupCountLN
647         txn.abort();
648         performRecoveryOperation();
649 
650         expectObsolete(file0, false);
651         expectObsolete(file1, false);
652         expectObsolete(file2, false);
653         expectObsolete(file3, true);
654         expectObsolete(file4, true);
655 
656         closeEnv(true);
657     }
658 
659     public void testInsertUpdate()
660         throws DatabaseException {
661 
662         openEnv();
663 
664         /* Insert and update key 0. */
665         Transaction txn = env.beginTransaction(null, null);
666         long file0 = doPut(0, txn);
667         long file1 = doPut(0, txn);
668         txn.commit();
669         performRecoveryOperation();
670 
671         expectObsolete(file0, true);
672         expectObsolete(file1, false);
673 
674         closeEnv(true);
675     }
676 
677     public void testInsertUpdateAbort()
678         throws DatabaseException {
679 
680         openEnv();
681 
682         /* Insert and update key 0. */
683         Transaction txn = env.beginTransaction(null, null);
684         long file0 = doPut(0, txn);
685         long file1 = doPut(0, txn);
686         txn.abort();
687         performRecoveryOperation();
688 
689         expectObsolete(file0, true);
690         expectObsolete(file1, true);
691 
692         closeEnv(true);
693     }
694 
695     public void testInsertUpdateDup()
696         throws DatabaseException {
697 
698         dups = true;
699         openEnv();
700 
701         /* Insert two key 0 dups and checkpoint. */
702         Transaction txn = env.beginTransaction(null, null);
703         long file0 = doPut(0, 0, txn); // 1st LN
704         long file2 = doPut(0, 1, txn); // 2nd LN
705         long file1 = file2 - 1;        // DupCountLN
706         txn.commit();
707         env.checkpoint(forceConfig);
708 
709         /* Insert and update {0, 2}. */
710         txn = env.beginTransaction(null, null);
711         long file3 = doPut(0, 2, txn);    // 3rd LN
712         long file4 = file3 + 1;           // DupCountLN
713         long file5 = doUpdate(0, 2, txn); // 4rd LN
714         txn.commit();
715         performRecoveryOperation();
716 
717         expectObsolete(file0, false);
718         expectObsolete(file1, true);
719         expectObsolete(file2, false);
720         expectObsolete(file3, true);
721         expectObsolete(file4, false);
722         expectObsolete(file5, false);
723 
724         closeEnv(true);
725     }
726 
727     public void testInsertUpdateDupAbort()
728         throws DatabaseException {
729 
730         dups = true;
731         openEnv();
732 
733         /* Insert two key 0 dups and checkpoint. */
734         Transaction txn = env.beginTransaction(null, null);
735         long file0 = doPut(0, 0, txn); // 1st LN
736         long file2 = doPut(0, 1, txn); // 2nd LN
737         long file1 = file2 - 1;        // DupCountLN
738         txn.commit();
739         env.checkpoint(forceConfig);
740 
741         /* Insert and update {0, 2}. */
742         txn = env.beginTransaction(null, null);
743         long file3 = doPut(0, 2, txn);    // 3rd LN
744         long file4 = file3 + 1;           // DupCountLN
745         long file5 = doUpdate(0, 2, txn); // 4rd LN
746         txn.abort();
747         performRecoveryOperation();
748 
749         expectObsolete(file0, false);
750         expectObsolete(file1, false);
751         expectObsolete(file2, false);
752         expectObsolete(file3, true);
753         expectObsolete(file4, true);
754         expectObsolete(file5, true);
755 
756         closeEnv(true);
757     }
758 
759     public void testInsertDelete()
760         throws DatabaseException {
761 
762         openEnv();
763 
764         /* Insert and update key 0. */
765         Transaction txn = env.beginTransaction(null, null);
766         long file0 = doPut(0, txn);
767         long file1 = doDelete(0, txn);
768         txn.commit();
769         performRecoveryOperation();
770 
771         expectObsolete(file0, true);
772         expectObsolete(file1, true);
773 
774         closeEnv(true);
775     }
776 
777     public void testInsertDeleteAbort()
778         throws DatabaseException {
779 
780         openEnv();
781 
782         /* Insert and update key 0. */
783         Transaction txn = env.beginTransaction(null, null);
784         long file0 = doPut(0, txn);
785         long file1 = doDelete(0, txn);
786         txn.abort();
787         performRecoveryOperation();
788 
789         expectObsolete(file0, true);
790         expectObsolete(file1, true);
791 
792         closeEnv(true);
793     }
794 
795     public void testInsertDeleteDup()
796         throws DatabaseException {
797 
798         dups = true;
799         openEnv();
800 
801         /* Insert two key 0 dups and checkpoint. */
802         Transaction txn = env.beginTransaction(null, null);
803         long file0 = doPut(0, 0, txn); // 1st LN
804         long file2 = doPut(0, 1, txn); // 2nd LN
805         long file1 = file2 - 1;        // DupCountLN
806         txn.commit();
807         env.checkpoint(forceConfig);
808 
809         /* Insert and delete {0, 2}. */
810         txn = env.beginTransaction(null, null);
811         long file3 = doPut(0, 2, txn);    // 3rd LN
812         long file4 = file3 + 1;           // DupCountLN
813         long file5 = doDelete(0, 2, txn); // 4rd LN
814         long file6 = file5 + 1;           // DupCountLN
815         txn.commit();
816         performRecoveryOperation();
817 
818         expectObsolete(file0, false);
819         expectObsolete(file1, true);
820         expectObsolete(file2, false);
821         expectObsolete(file3, true);
822         expectObsolete(file4, true);
823         expectObsolete(file5, true);
824         expectObsolete(file6, false);
825 
826         closeEnv(true);
827     }
828 
829     public void testInsertDeleteDupAbort()
830         throws DatabaseException {
831 
832         dups = true;
833         openEnv();
834 
835         /* Insert two key 0 dups and checkpoint. */
836         Transaction txn = env.beginTransaction(null, null);
837         long file0 = doPut(0, 0, txn); // 1st LN
838         long file2 = doPut(0, 1, txn); // 2nd LN
839         long file1 = file2 - 1;        // DupCountLN
840         txn.commit();
841         env.checkpoint(forceConfig);
842 
843         /* Insert and delete {0, 2} and abort. */
844         txn = env.beginTransaction(null, null);
845         long file3 = doPut(0, 2, txn);    // 3rd LN
846         long file4 = file3 + 1;           // DupCountLN
847         long file5 = doDelete(0, 2, txn); // 4rd LN
848         long file6 = file5 + 1;           // DupCountLN
849         txn.abort();
850         performRecoveryOperation();
851 
852         expectObsolete(file0, false);
853         expectObsolete(file1, false);
854         expectObsolete(file2, false);
855         expectObsolete(file3, true);
856         expectObsolete(file4, true);
857         expectObsolete(file5, true);
858         expectObsolete(file6, true);
859 
860         closeEnv(true);
861     }
862 
863     public void testUpdateUpdate()
864         throws DatabaseException {
865 
866         openEnv();
867 
868         /* Insert key 0 and checkpoint. */
869         long file0 = doPut(0, true);
870         env.checkpoint(forceConfig);
871 
872         /* Update key 0 twice. */
873         Transaction txn = env.beginTransaction(null, null);
874         long file1 = doPut(0, txn);
875         long file2 = doPut(0, txn);
876         txn.commit();
877         performRecoveryOperation();
878 
879         expectObsolete(file0, true);
880         expectObsolete(file1, true);
881         expectObsolete(file2, false);
882 
883         closeEnv(true);
884     }
885 
886     public void testUpdateUpdateAbort()
887         throws DatabaseException {
888 
889         openEnv();
890 
891         /* Insert key 0 and checkpoint. */
892         long file0 = doPut(0, true);
893         env.checkpoint(forceConfig);
894 
895         /* Update key 0 twice and abort. */
896         Transaction txn = env.beginTransaction(null, null);
897         long file1 = doPut(0, txn);
898         long file2 = doPut(0, txn);
899         txn.abort();
900         performRecoveryOperation();
901 
902         expectObsolete(file0, false);
903         expectObsolete(file1, true);
904         expectObsolete(file2, true);
905 
906         closeEnv(true);
907     }
908 
909     public void testUpdateUpdateDup()
910         throws DatabaseException {
911 
912         dups = true;
913         openEnv();
914 
915         /* Insert two key 0 dups and checkpoint. */
916         Transaction txn = env.beginTransaction(null, null);
917         long file0 = doPut(0, 0, txn); // 1st LN
918         long file2 = doPut(0, 1, txn); // 2nd LN
919         long file1 = file2 - 1;        // DupCountLN
920         txn.commit();
921         env.checkpoint(forceConfig);
922 
923         /* Update {0, 1} twice. */
924         txn = env.beginTransaction(null, null);
925         long file3 = doUpdate(0, 1, txn); // 3rd LN
926         long file4 = doUpdate(0, 1, txn); // 4rd LN
927         txn.commit();
928         performRecoveryOperation();
929 
930         expectObsolete(file0, false);
931         expectObsolete(file1, false);
932         expectObsolete(file2, true);
933         expectObsolete(file3, true);
934         expectObsolete(file4, false);
935 
936         closeEnv(true);
937     }
938 
939     public void testUpdateUpdateDupAbort()
940         throws DatabaseException {
941 
942         dups = true;
943         openEnv();
944 
945         /* Insert two key 0 dups and checkpoint. */
946         Transaction txn = env.beginTransaction(null, null);
947         long file0 = doPut(0, 0, txn); // 1st LN
948         long file2 = doPut(0, 1, txn); // 2nd LN
949         long file1 = file2 - 1;        // DupCountLN
950         txn.commit();
951         env.checkpoint(forceConfig);
952 
953         /* Update {0, 1} twice and abort. */
954         txn = env.beginTransaction(null, null);
955         long file3 = doUpdate(0, 1, txn); // 3rd LN
956         long file4 = doUpdate(0, 1, txn); // 4rd LN
957         txn.abort();
958         performRecoveryOperation();
959 
960         expectObsolete(file0, false);
961         expectObsolete(file1, false);
962         expectObsolete(file2, false);
963         expectObsolete(file3, true);
964         expectObsolete(file4, true);
965 
966         closeEnv(true);
967     }
968 
969     public void testUpdateDelete()
970         throws DatabaseException {
971 
972         openEnv();
973 
974         /* Insert key 0 and checkpoint. */
975         long file0 = doPut(0, true);
976         env.checkpoint(forceConfig);
977 
978         /* Update and delete key 0. */
979         Transaction txn = env.beginTransaction(null, null);
980         long file1 = doPut(0, txn);
981         long file2 = doDelete(0, txn);
982         txn.commit();
983         performRecoveryOperation();
984 
985         expectObsolete(file0, true);
986         expectObsolete(file1, true);
987         expectObsolete(file2, true);
988 
989         closeEnv(true);
990     }
991 
992     public void testUpdateDeleteAbort()
993         throws DatabaseException {
994 
995         openEnv();
996 
997         /* Insert key 0 and checkpoint. */
998         long file0 = doPut(0, true);
999         env.checkpoint(forceConfig);
1000
1001        /* Update and delete key 0 and abort. */
1002        Transaction txn = env.beginTransaction(null, null);
1003        long file1 = doPut(0, txn);
1004        long file2 = doDelete(0, txn);
1005        txn.abort();
1006        performRecoveryOperation();
1007
1008        expectObsolete(file0, false);
1009        expectObsolete(file1, true);
1010        expectObsolete(file2, true);
1011
1012        closeEnv(true);
1013    }
1014
1015    public void testUpdateDeleteDup()
1016        throws DatabaseException {
1017
1018        dups = true;
1019        openEnv();
1020
1021        /* Insert two key 0 dups and checkpoint. */
1022        Transaction txn = env.beginTransaction(null, null);
1023        long file0 = doPut(0, 0, txn); // 1st LN
1024        long file2 = doPut(0, 1, txn); // 2nd LN
1025        long file1 = file2 - 1;        // DupCountLN
1026        txn.commit();
1027        env.checkpoint(forceConfig);
1028
1029        /* Update and delete {0, 1}. */
1030        txn = env.beginTransaction(null, null);
1031        long file3 = doUpdate(0, 1, txn); // 3rd LN
1032        long file4 = doDelete(0, 1, txn); // 4rd LN
1033        long file5 = file4 + 1;           // DupCountLN
1034        txn.commit();
1035        performRecoveryOperation();
1036
1037        expectObsolete(file0, false);
1038        expectObsolete(file1, true);
1039        expectObsolete(file2, true);
1040        expectObsolete(file3, true);
1041        expectObsolete(file4, true);
1042        expectObsolete(file5, false);
1043
1044        closeEnv(true);
1045    }
1046
1047    public void testUpdateDeleteDupAbort()
1048        throws DatabaseException {
1049
1050        dups = true;
1051        openEnv();
1052
1053        /* Insert two key 0 dups and checkpoint. */
1054        Transaction txn = env.beginTransaction(null, null);
1055        long file0 = doPut(0, 0, txn); // 1st LN
1056        long file2 = doPut(0, 1, txn); // 2nd LN
1057        long file1 = file2 - 1;        // DupCountLN
1058        txn.commit();
1059        env.checkpoint(forceConfig);
1060
1061        /* Update and delete {0, 1} and abort. */
1062        txn = env.beginTransaction(null, null);
1063        long file3 = doUpdate(0, 1, txn); // 3rd LN
1064        long file4 = doDelete(0, 1, txn); // 4rd LN
1065        long file5 = file4 + 1;           // DupCountLN
1066        txn.abort();
1067        performRecoveryOperation();
1068
1069        expectObsolete(file0, false);
1070        expectObsolete(file1, false);
1071        expectObsolete(file2, false);
1072        expectObsolete(file3, true);
1073        expectObsolete(file4, true);
1074        expectObsolete(file5, true);
1075
1076        closeEnv(true);
1077    }
1078
1079    public void testTruncate()
1080        throws DatabaseException {
1081
1082        truncateOrRemove(true, true);
1083    }
1084
1085    public void testTruncateAbort()
1086        throws DatabaseException {
1087
1088        truncateOrRemove(true, false);
1089    }
1090
1091    public void testRemove()
1092        throws DatabaseException {
1093
1094        truncateOrRemove(false, true);
1095    }
1096
1097    public void testRemoveAbort()
1098        throws DatabaseException {
1099
1100        truncateOrRemove(false, false);
1101    }
1102
1103    /**
1104     * @deprecated use of Database.truncate
1105     */
1106    private void truncateOrRemove(boolean truncate, boolean commit)
1107        throws DatabaseException {
1108
1109        openEnv();
1110
1111        /* Insert 3 keys and checkpoint. */
1112        Transaction txn = env.beginTransaction(null, null);
1113        long file0 = doPut(0, txn);
1114        long file1 = doPut(1, txn);
1115        long file2 = doPut(2, txn);
1116        txn.commit();
1117        env.checkpoint(forceConfig);
1118
1119        /* Truncate. */
1120        txn = env.beginTransaction(null, null);
1121        if (truncate) {
1122            int count = db.truncate(txn, true);
1123            assertEquals(3, count);
1124        } else {
1125            db.close();
1126            db = null;
1127            env.removeDatabase(txn, DB_NAME);
1128        }
1129        if (commit) {
1130            txn.commit();
1131        } else {
1132            txn.abort();
1133        }
1134        performRecoveryOperation();
1135
1136        expectObsolete(file0, commit);
1137        expectObsolete(file1, commit);
1138        expectObsolete(file2, commit);
1139
1140        closeEnv(true);
1141    }
1142
1143    private void expectObsolete(long file, boolean obsolete)
1144        throws DatabaseException {
1145
1146        FileSummary summary = getSummary(file);
1147        assertEquals("totalLNCount",
1148                     1, summary.totalLNCount);
1149        assertEquals("obsoleteLNCount",
1150                     obsolete ? 1 : 0, summary.obsoleteLNCount);
1151
1152        if (obsolete) {
1153            if (expectAccurateObsoleteLNSize()) {
1154                assertTrue(summary.obsoleteLNSize > 0);
1155                assertEquals(1, summary.obsoleteLNSizeCounted);
1156            }
1157            /* If we counted the size, make sure it is the actual LN size. */
1158            if (summary.obsoleteLNSize > 0) {
1159                assertEquals(getLNSize(file), summary.obsoleteLNSize);
1160            }
1161        } else {
1162            assertEquals(0, summary.obsoleteLNSize);
1163            assertEquals(0, summary.obsoleteLNSizeCounted);
1164        }
1165    }
1166
1167    /**
1168     * If an LN is obsolete, expect the size to be counted unless we ran
1169     * recovery and we did NOT configure fetchObsoleteSize=true.  In that
1170     * case, the size may or may not be counted depending on how the redo
1171     * or undo was processed during reocvery.
1172     */
1173    private boolean expectAccurateObsoleteLNSize() {
1174        return fetchObsoleteSize || !OP_RECOVER.equals(operation);
1175    }
1176
1177    private long doPut(int key, boolean commit)
1178        throws DatabaseException {
1179
1180        Transaction txn = env.beginTransaction(null, null);
1181        long file = doPut(key, txn);
1182        if (commit) {
1183            txn.commit();
1184        } else {
1185            txn.abort();
1186        }
1187        return file;
1188    }
1189
1190    private long doPut(int key, Transaction txn)
1191        throws DatabaseException {
1192
1193        return doPut(key, key, txn);
1194    }
1195
1196    private long doPut(int key, int data, Transaction txn)
1197        throws DatabaseException {
1198
1199        Cursor cursor = db.openCursor(txn, null);
1200        IntegerBinding.intToEntry(key, keyEntry);
1201        IntegerBinding.intToEntry(data, dataEntry);
1202        cursor.put(keyEntry, dataEntry);
1203        long file = getFile(cursor);
1204        cursor.close();
1205        return file;
1206    }
1207
1208    private long doUpdate(int key, int data, Transaction txn)
1209        throws DatabaseException {
1210
1211        Cursor cursor = db.openCursor(txn, null);
1212        IntegerBinding.intToEntry(key, keyEntry);
1213        IntegerBinding.intToEntry(data, dataEntry);
1214        assertEquals(OperationStatus.SUCCESS,
1215                     cursor.getSearchBoth(keyEntry, dataEntry, null));
1216        cursor.putCurrent(dataEntry);
1217        long file = getFile(cursor);
1218        cursor.close();
1219        return file;
1220    }
1221
1222    private long doDelete(int key, boolean commit)
1223        throws DatabaseException {
1224
1225        Transaction txn = env.beginTransaction(null, null);
1226        long file = doDelete(key, txn);
1227        if (commit) {
1228            txn.commit();
1229        } else {
1230            txn.abort();
1231        }
1232        return file;
1233    }
1234
1235    private long doDelete(int key, Transaction txn)
1236        throws DatabaseException {
1237
1238        Cursor cursor = db.openCursor(txn, null);
1239        IntegerBinding.intToEntry(key, keyEntry);
1240        assertEquals(OperationStatus.SUCCESS,
1241                     cursor.getSearchKey(keyEntry, dataEntry, null));
1242        cursor.delete();
1243        long file = getFile(cursor);
1244        cursor.close();
1245        return file;
1246    }
1247
1248    private long doDelete(int key, int data, Transaction txn)
1249        throws DatabaseException {
1250
1251        Cursor cursor = db.openCursor(txn, null);
1252        IntegerBinding.intToEntry(key, keyEntry);
1253        IntegerBinding.intToEntry(data, dataEntry);
1254        assertEquals(OperationStatus.SUCCESS,
1255                     cursor.getSearchBoth(keyEntry, dataEntry, null));
1256        cursor.delete();
1257        long file = getFile(cursor);
1258        cursor.close();
1259        return file;
1260    }
1261
1262    /**
1263     * Checkpoint, recover, or do nothing.
1264     */
1265    private void performRecoveryOperation()
1266        throws DatabaseException {
1267
1268        if (OP_NONE.equals(operation)) {
1269            /* Compress to count deleted LNs. */
1270            env.compress();
1271        } else if (OP_CHECKPOINT.equals(operation)) {
1272            /* Compress before checkpointing to count deleted LNs. */
1273            env.compress();
1274            env.checkpoint(forceConfig);
1275        } else if (OP_RECOVER.equals(operation)) {
1276            closeEnv(false);
1277            openEnv();
1278            /* Compress after recovery to count deleted LNs. */
1279            env.compress();
1280        } else {
1281            assert false : operation;
1282        }
1283    }
1284
1285    /**
1286     * Gets the file of the LSN at the cursor position, using internal methods.
1287     * Also check that the file number is greater than the last file returned,
1288     * to ensure that we're filling a file every time we write.
1289     */
1290    private long getFile(Cursor cursor)
1291        throws DatabaseException {
1292
1293        long file = CleanerTestUtils.getLogFile(this, cursor);
1294        assert file > lastFileSeen;
1295        lastFileSeen = file;
1296        return file;
1297    }
1298
1299    /**
1300     * Returns the utilization summary for a given log file.
1301     */
1302    private FileSummary getSummary(long file)
1303        throws DatabaseException {
1304
1305        return (FileSummary)
1306            DbInternal.envGetEnvironmentImpl(env)
1307                      .getUtilizationProfile()
1308                      .getFileSummaryMap(true)
1309                      .get(new Long  (file));
1310    }
1311
1312    /**
1313     * Peek into the file to get the total size of the first entry past the
1314     * file header, which is known to be the LN log entry.
1315     */
1316    private int getLNSize(long file)
1317        throws DatabaseException {
1318
1319        try {
1320            long offset = LogManager.HEADER_BYTES + FileHeader.entrySize();
1321            long lsn = DbLsn.makeLsn(file, offset);
1322            LogManager lm =
1323                DbInternal.envGetEnvironmentImpl(env).getLogManager();
1324            LogSource src = lm.getLogSource(lsn);
1325            ByteBuffer   buf =
1326                src.getBytes(offset + LogManager.HEADER_SIZE_OFFSET);
1327            int size = LogUtils.readInt(buf);
1328            src.release();
1329            return size + LogManager.HEADER_BYTES;
1330        } catch (IOException   e) {
1331            throw new DatabaseException(e);
1332        }
1333    }
1334}
1335
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags