zzz/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

- Free unused span continuation pages (big space savings)

Browse Source 
- Less data copying during on-disk searches
This commit is contained in:
zzz
2011-03-28 04:57:16 +00:00
parent 991acd3917
commit 9fcb20a7bd
3 changed files with 115 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
core/java/src/net/metanotion/io/block/BlockFile.java
View File

@@ -45,6 +45,7 @@ import net.metanotion.io.data.NullBytes;
import net.metanotion.io.data.StringBytes;
import net.metanotion.io.data.UTF8StringBytes;
import net.metanotion.io.block.index.BSkipList;
import net.metanotion.io.block.index.BSkipSpan;
import net.metanotion.util.skiplist.SkipIterator;
import net.metanotion.util.skiplist.SkipList;
@@ -132,6 +133,16 @@ public class BlockFile {
}
}
/**
* Write bytes
* This will allocate additional continuation pages as necessary.
*
* @param data data to write
* @param page current page
* @param curPageOff in (current) and out (new) parameter at index 0
* @param nextPage in (current) and out (new) parameter at index 0
* @return current page
*/
public int writeMultiPageData(byte[] data, int page, int[] curPageOff, int[] nextPage) throws IOException {
int pageCounter = curPageOff[0];
int curNextPage = nextPage[0];
@@ -151,9 +162,11 @@ public class BlockFile {
}
BlockFile.pageSeek(this.file, curNextPage);
curPage = curNextPage;
this.file.skipBytes(4); // skip magic
int magic = this.file.readInt();
if (magic != MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
curNextPage = this.file.readUnsignedInt();
pageCounter = 8;
pageCounter = BSkipSpan.CONT_HEADER_LEN;
len = PAGESIZE - pageCounter;
}
this.file.write(data, dct, Math.min(len, data.length - dct));
@@ -165,25 +178,35 @@ public class BlockFile {
return curPage;
}
/**
* Read bytes
*
* @param arr fill this array fully with data
* @param page current page
* @param curPageOff in (current) and out (new) parameter at index 0
* @param nextPage in (current) and out (new) parameter at index 0
* @return current page
*/
public int readMultiPageData(byte[] arr, int page, int[] curPageOff, int[] nextPage) throws IOException {
int pageCounter = curPageOff[0];
int curNextPage = nextPage[0];
int curPage = page;
int dct = 0;
int res;
while(dct < arr.length) {
int len = PAGESIZE - pageCounter;
if(len <= 0) {
if (curNextPage <= 0)
throw new IOException("not enough pages to read data still need " + (arr.length - dct));
BlockFile.pageSeek(this.file, curNextPage);
int magic = this.file.readInt();
if (magic != MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
curPage = curNextPage;
curNextPage = this.file.readUnsignedInt();
pageCounter = 8;
pageCounter = BSkipSpan.CONT_HEADER_LEN;
len = PAGESIZE - pageCounter;
}
res = this.file.read(arr, dct, Math.min(len, arr.length - dct));
int res = this.file.read(arr, dct, Math.min(len, arr.length - dct));
if(res == -1) { throw new IOException(); }
pageCounter += Math.min(len, arr.length - dct);
dct += res;
@@ -193,6 +216,45 @@ public class BlockFile {
return curPage;
}
/**
* Skip length bytes
* The same as readMultiPageData() without returning a result
*
* @param length number of bytes to skip
* @param page current page
* @param curPageOff in (current) and out (new) parameter at index 0
* @param nextPage in (current) and out (new) parameter at index 0
* @return current page
*/
public int skipMultiPageBytes(int length, int page, int[] curPageOff, int[] nextPage) throws IOException {
int pageCounter = curPageOff[0];
int curNextPage = nextPage[0];
int curPage = page;
int dct = 0;
while(dct < length) {
int len = PAGESIZE - pageCounter;
if(len <= 0) {
if (curNextPage <= 0)
throw new IOException("not enough pages to skip");
BlockFile.pageSeek(this.file, curNextPage);
int magic = this.file.readInt();
if (magic != MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
curPage = curNextPage;
curNextPage = this.file.readUnsignedInt();
pageCounter = BSkipSpan.CONT_HEADER_LEN;
len = PAGESIZE - pageCounter;
}
int res = Math.min(len, length - dct);
this.file.skipBytes(res);
pageCounter += res;
dct += res;
}
nextPage[0] = curNextPage;
curPageOff[0] = pageCounter;
return curPage;
}
public BlockFile(RandomAccessInterface rai) throws IOException { this(rai, false); }
public BlockFile(RandomAccessFile raf) throws IOException { this(new RAIFile(raf), false); }
public BlockFile(RandomAccessFile raf, boolean init) throws IOException { this(new RAIFile(raf), init); }

51
core/java/src/net/metanotion/io/block/index/BSkipSpan.java
View File

@@ -36,6 +36,8 @@ import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipSpan;
import net.i2p.util.Log;
/**
* On-disk format:
*
@@ -59,7 +61,7 @@ import net.metanotion.util.skiplist.SkipSpan;
public class BSkipSpan extends SkipSpan {
protected static final int MAGIC = 0x5370616e; // "Span"
protected static final int HEADER_LEN = 20;
protected static final int CONT_HEADER_LEN = 8;
public static final int CONT_HEADER_LEN = 8;
protected final BlockFile bf;
private final BSkipList bsl;
protected int page;
@@ -102,21 +104,33 @@ public class BSkipSpan extends SkipSpan {
try {
int curPage = overflowPage;
bf.freePage(page);
while(curPage != 0) {
BlockFile.pageSeek(bf.file, curPage);
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
int next = bf.file.readUnsignedInt();
bf.freePage(curPage);
curPage = next;
}
freeContinuationPages(curPage);
} catch (IOException ioe) {
BlockFile.log.error("Error freeing " + this, ioe);
}
bsl.spanHash.remove(this.page);
}
/**
* Free a chain of continuation pages
* @param curPage the first page to be freed, if 0 this does nothing.
* @return number freed
*/
private int freeContinuationPages(int curPage) throws IOException {
int rv = 0;
while(curPage > 0) {
BlockFile.pageSeek(bf.file, curPage);
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
int next = bf.file.readUnsignedInt();
bf.freePage(curPage);
curPage = next;
rv++;
}
return rv;
}
public void flush() {
fflush();
}
@@ -193,10 +207,24 @@ public class BSkipSpan extends SkipSpan {
curPage = bf.writeMultiPageData(keyData, curPage, pageCounter, curNextPage);
curPage = bf.writeMultiPageData(valData, curPage, pageCounter, curNextPage);
}
// FIXME why seek and rescan the overflow page?
BlockFile.pageSeek(bf.file, this.page);
bf.file.skipBytes(4); // skip magic
this.overflowPage = bf.file.readUnsignedInt();
if (curNextPage[0] != 0) {
// free extra continuation pages
BlockFile.pageSeek(bf.file, curPage);
bf.file.skipBytes(4); // skip magic
bf.file.writeInt(0);
if (curPage == this.page)
this.overflowPage = 0;
try {
int freed = freeContinuationPages(curNextPage[0]);
if (BlockFile.log.shouldLog(Log.INFO))
BlockFile.log.info("Freed " + freed + " continuation pages");
} catch (IOException ioe) {
BlockFile.log.error("Error freeing " + this, ioe);
}
}
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
// FIXME can't get there from here
//bsl.size -= fail;
@@ -292,6 +320,7 @@ public class BSkipSpan extends SkipSpan {
continue;
}
}
// free any excess overflow pages?
if (fail > 0) {
BlockFile.log.error("Repairing corruption of " + fail + " entries");
if (flushOnError)

13
core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
View File

@@ -162,7 +162,6 @@ public class IBSkipSpan extends BSkipSpan {
*/
private Object getData(Comparable key) throws IOException {
seekData();
int ksz, vsz;
int curPage = this.page;
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
@@ -180,16 +179,16 @@ public class IBSkipSpan extends BSkipSpan {
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = CONT_HEADER_LEN;
}
ksz = this.bf.file.readUnsignedShort();
vsz = this.bf.file.readUnsignedShort();
int ksz = this.bf.file.readUnsignedShort();
int vsz = this.bf.file.readUnsignedShort();
pageCounter[0] +=4;
byte[] k = new byte[ksz];
byte[] v = new byte[vsz];
curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
//System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz);
Comparable ckey = (Comparable) this.keySer.construct(k);
if (ckey == null) {
// skip the value and keep going
curPage = this.bf.skipMultiPageBytes(vsz, curPage, pageCounter, curNextPage);
BlockFile.log.error("Null deserialized key in entry " + i + " page " + curPage);
fail++;
continue;
@@ -197,6 +196,8 @@ public class IBSkipSpan extends BSkipSpan {
int diff = ckey.compareTo(key);
if (diff == 0) {
//System.err.println("Found " + key + " at " + i + " (first: " + this.firstKey + ')');
byte[] v = new byte[vsz];
curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
Object rv = this.valSer.construct(v);
if (rv == null) {
BlockFile.log.error("Null deserialized value in entry " + i + " page " + curPage +
@@ -213,6 +214,8 @@ public class IBSkipSpan extends BSkipSpan {
repair(fail);
return null;
}
// skip the value and keep going
curPage = this.bf.skipMultiPageBytes(vsz, curPage, pageCounter, curNextPage);
}
//System.err.println("NOT Found " + key + " at end (first: " + this.firstKey + ')');
if (fail > 0)