0.9.2

(and updated en po files)

.mtn-ignore

@@ -17,10 +17,10 @@ _jsp\.java$
 \.war$
 \.zip$
 ^\.
-^build/
+^build
 ^pkg-temp/
 ~$
-/build/
+/build
 /classes/
 ^debian/copyright
 override.properties

.tx/config

@@ -2,14 +2,17 @@
 source_file = apps/i2ptunnel/locale/messages_en.po
 source_lang = en
 trans.ar = apps/i2ptunnel/locale/messages_ar.po
+trans.cs = apps/i2ptunnel/locale/messages_cs.po
 trans.da = apps/i2ptunnel/locale/messages_da.po
 trans.de = apps/i2ptunnel/locale/messages_de.po
 trans.es = apps/i2ptunnel/locale/messages_es.po
 trans.fr = apps/i2ptunnel/locale/messages_fr.po
+trans.hu = apps/i2ptunnel/locale/messages_hu.po
 trans.it = apps/i2ptunnel/locale/messages_it.po
 trans.nl = apps/i2ptunnel/locale/messages_nl.po
 trans.ru = apps/i2ptunnel/locale/messages_ru.po
 trans.sv_SE = apps/i2ptunnel/locale/messages_sv.po
+trans.uk_UA = apps/i2ptunnel/locale/messages_uk.po
 trans.vi = apps/i2ptunnel/locale/messages_vi.po
 trans.zh_CN = apps/i2ptunnel/locale/messages_zh.po
 
@@ -17,11 +20,15 @@ trans.zh_CN = apps/i2ptunnel/locale/messages_zh.po
 source_file = apps/routerconsole/locale/messages_en.po
 source_lang = en
 trans.ar = apps/routerconsole/locale/messages_ar.po
+trans.cs = apps/routerconsole/locale/messages_cs.po
 trans.da = apps/routerconsole/locale/messages_da.po
 trans.de = apps/routerconsole/locale/messages_de.po
+trans.el = apps/routerconsole/locale/messages_el.po
 trans.es = apps/routerconsole/locale/messages_es.po
+trans.et_EE = apps/routerconsole/locale/messages_ee.po
 trans.fi = apps/routerconsole/locale/messages_fi.po
 trans.fr = apps/routerconsole/locale/messages_fr.po
+trans.hu = apps/routerconsole/locale/messages_hu.po
 trans.it = apps/routerconsole/locale/messages_it.po
 trans.nl = apps/routerconsole/locale/messages_nl.po
 trans.pl = apps/routerconsole/locale/messages_pl.po
@@ -35,9 +42,11 @@ trans.zh_CN = apps/routerconsole/locale/messages_zh.po
 source_file = apps/i2psnark/locale/messages_en.po
 source_lang = en
 trans.ar = apps/i2psnark/locale/messages_ar.po
+trans.cs = apps/i2psnark/locale/messages_cs.po
 trans.de = apps/i2psnark/locale/messages_de.po
 trans.es = apps/i2psnark/locale/messages_es.po
 trans.fr = apps/i2psnark/locale/messages_fr.po
+trans.hu = apps/i2psnark/locale/messages_hu.po
 trans.it = apps/i2psnark/locale/messages_it.po
 trans.nl = apps/i2psnark/locale/messages_nl.po
 trans.pl = apps/i2psnark/locale/messages_pl.po
@@ -51,10 +60,13 @@ trans.zh_CN = apps/i2psnark/locale/messages_zh.po
 source_file = apps/susidns/locale/messages_en.po
 source_lang = en
 trans.ar = apps/susidns/locale/messages_ar.po
+trans.cs = apps/susidns/locale/messages_cs.po
 trans.da = apps/susidns/locale/messages_da.po
 trans.de = apps/susidns/locale/messages_de.po
+trans.el = apps/susidns/locale/messages_el.po
 trans.es = apps/susidns/locale/messages_es.po
 trans.fr = apps/susidns/locale/messages_fr.po
+trans.hu = apps/susidns/locale/messages_hu.po
 trans.it = apps/susidns/locale/messages_it.po
 trans.nl = apps/susidns/locale/messages_nl.po
 trans.pl = apps/susidns/locale/messages_pl.po
@@ -68,10 +80,13 @@ trans.zh_CN = apps/susidns/locale/messages_zh.po
 source_file = apps/desktopgui/locale/messages_en.po
 source_lang = en
 trans.ar = apps/desktopgui/locale/messages_ar.po
+trans.cs = apps/desktopgui/locale/messages_cs.po
 trans.da = apps/desktopgui/locale/messages_da.po
 trans.de = apps/desktopgui/locale/messages_de.po
+trans.el = apps/desktopgui/locale/messages_el.po
 trans.es = apps/desktopgui/locale/messages_es.po
 trans.fr = apps/desktopgui/locale/messages_fr.po
+trans.hu = apps/desktopgui/locale/messages_hu.po
 trans.it = apps/desktopgui/locale/messages_it.po
 trans.nl = apps/desktopgui/locale/messages_nl.po
 trans.pl = apps/desktopgui/locale/messages_pl.po
@@ -84,15 +99,34 @@ trans.zh_CN = apps/desktopgui/locale/messages_zh.po
 [I2P.susimail]
 source_file = apps/susimail/locale/messages_en.po
 source_lang = en
+trans.cs = apps/susimail/locale/messages_cs.po
 trans.de = apps/susimail/locale/messages_de.po
 trans.es = apps/susimail/locale/messages_es.po
 trans.fr = apps/susimail/locale/messages_fr.po
+trans.hu = apps/susimail/locale/messages_hu.po
+trans.it = apps/susimail/locale/messages_it.po
 trans.nl = apps/susimail/locale/messages_nl.po
+trans.ru = apps/susimail/locale/messages_ru.po
 trans.sv_SE = apps/susimail/locale/messages_sv.po
 trans.pl = apps/susimail/locale/messages_pl.po
+trans.uk_UA = apps/susimail/locale/messages_uk.po
 trans.vi = apps/susimail/locale/messages_vi.po
 trans.zh_CN = apps/susimail/locale/messages_zh.po
 
+[I2P.debconf]
+source_file = debian/po/templates.pot
+source_lang = en
+trans.cs = debian/po/cs.po
+trans.de = debian/po/de.po
+trans.el = debian/po/el.po
+trans.es = debian/po/es.po
+trans.it = debian/po/it.po
+trans.hu = debian/po/hu.po
+trans.pl = debian/po/pl.po
+trans.ru = debian/po/ru.po
+trans.sv_SE = debian/po/sv.po
+trans.uk_UA = debian/po/uk.po
+
 [main]
 host = http://www.transifex.net
 

INSTALL.txt

@@ -4,33 +4,54 @@ Prerequisites to build from source:
 	Java SDK (preferably Oracle/Sun or OpenJDK) 1.6.0 or higher
           Non-linux operating systems and JVMs: See http://trac.i2p2.de/wiki/java
 	Apache Ant 1.7.0 or higher
-	Optional, For multilanguage support: The xgettext, msgfmt, and msgmerge tools installed
+	The xgettext, msgfmt, and msgmerge tools installed
 	from the GNU gettext package http://www.gnu.org/software/gettext/
 
 To build and install I2P from source, you must first build
 and package up the appropriate installer by running:
 
-  ant pkg
+  	ant pkg
+
+On non-x86, use one of the following instead:
+	ant installer-linux
+	ant installer-freebsd
+	ant installer-osx
+
 
 
 This will produce a few key files:
 * install.jar:    the GUI and console installer
 * i2pinstall.exe: the GUI and console installer wrapped for cross-platform execution
+                  (only created with ant pkg)
 * i2pupdate.zip:  the update package
+                  (only created with ant pkg)
 
 From there, you can run the headless (console mode) installer:
   java -jar i2pinstall.exe -console
+or
+  java -jar i2pinstall.jar -console
 
 Or run the GUI installer:
   java -jar i2pinstall.exe
+or
+  java -jar i2pinstall.jar
+or on Windows, just double-click on i2pinstall.exe.
 
 Or move the i2pupdate.zip file into an existing installation directory and restart.
 
 To start I2P:
    (*nix): sh i2prouter start
-   (win*): I2P.exe
+   (win*): I2P.exe or i2prouter.bat
    (non-x86 platforms PPC, ARM, etc): sh runplain.sh
 
+To install I2P as a system service:
+   (*nix) sh i2prouter install
+   (win*) install_i2p_service_winnt.bat
+
+To uninstall I2P as a system service:
+   (*nix) sh i2prouter remove
+   (win*) uninstall_i2p-service_winnt.bat
+
 To stop I2P (gracefully):
    lynx http://localhost:7657/summaryframe (click "Shutdown")
 

LICENSE.txt

@@ -2,7 +2,7 @@ This product includes both public domain code and licensed code as described bel
 For all code, unless otherwise stated in the appropriate license, the following applies:
 
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -79,11 +79,13 @@ Public domain except as listed below:
    From freenet
    See licenses/LICENSE-GPLv2.txt
 
-   UPnP subsystem 1.7:
-   Copyright (C) 2003-2006 Satoshi Konno
+   UPnP subsystem (CyberLink) 2.1:
+   Copyright (C) 2003-2010 Satoshi Konno
    See licenses/LICENSE-UPnP.txt
 
-   GeoIP data free to use, courtesy http://ip-to-country.webhosting.info/
+   GeoIP: GeoLite databases are licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License.
+   http://creativecommons.org/licenses/by-sa/3.0/
+   This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com/
 
 
 Installer:
@@ -92,34 +94,34 @@ Installer:
    See licenses/LICENSE-Launch4j.txt (in binary packages)
    See installer/lib/launch4j/LICENSE.txt (in source packages)
    The following projects are used by Launch4j...
-	MinGW binutils (http://www.mingw.org/)
+        MinGW binutils (http://www.mingw.org/)
 
-	Commons BeanUtils (http://jakarta.apache.org/commons/beanutils/)
+        Commons BeanUtils (http://jakarta.apache.org/commons/beanutils/)
 
- 	Commons Logging (http://jakarta.apache.org/commons/logging/)
-        See licenses/LICENSE-Apache1.1.txt
+        Commons Logging (http://jakarta.apache.org/commons/logging/)
+        See licenses/LICENSE-Apache2.0.txt
         See licenses/NOTICE-Commons-Logging.txt
 
-	XStream (http://xstream.codehaus.org/)
+        XStream (http://xstream.codehaus.org/)
         Copyright (c) 2003-2004, Joe Walnes
         See licenses/LICENSE-XStream.txt
 
-	JGoodies Forms (http://www.jgoodies.com/freeware/forms/)
+        JGoodies Forms (http://www.jgoodies.com/freeware/forms/)
         Copyright (c) 2002-2004 JGoodies Karsten Lentzsch. All rights reserved.
         See licenses/LICENSE-JGoodies-Forms.txt
 
-	JGoodies Looks (http://www.jgoodies.com/freeware/looks/)
+        JGoodies Looks (http://www.jgoodies.com/freeware/looks/)
         Copyright (c) 2003 JGoodies Karsten Lentzsch. All rights reserved.
         See licenses/LICENSE-JGoodies-Looks.txt
 
-	Foxtrot (http://foxtrot.sourceforge.net/)
+        Foxtrot (http://foxtrot.sourceforge.net/)
         Copyright (c) 2002, Simone Bordet & Marco Cravero. All rights reserved.
         See licenses/LICENSE-Foxtrot.txt
 
-	Nuvola Icon Theme (http://www.icon-king.com)
+        Nuvola Icon Theme (http://www.icon-king.com)
         See licenses/LICENSE-LGPLv2.1.txt
 
-	Forms were created using Abeille Forms Designer (https://abeille.dev.java.net/)
+        Forms were created using Abeille Forms Designer (https://abeille.dev.java.net/)
 
    Izpack 4.3.0:
    Copyright (c) 2001-2008 Julien Ponge
@@ -127,7 +129,7 @@ Installer:
 
 
 
-Java Service Wrapper Community Edition 32-bit 3.5.9:
+Java Service Wrapper Community Edition 32-bit 3.5.13:
    Copyright (C) 1999-2011 Tanuki Software, Ltd. All Rights Reserved.
    See licenses/LICENSE-Wrapper.txt
 
@@ -135,10 +137,6 @@ Java Service Wrapper Community Edition 32-bit 3.5.9:
 Jbigi Libraries (jbigi.jar):
    JNI code public domain.
 
-   GMP 4.1.3 / 4.1.4:
-   Copyright 1991, 1996, 1999, 2000 Free Software Foundation, Inc.
-   See licenses/LICENSE-LGPLv2.1.txt
-
    GMP 4.3.2 / 5.0.2:
    Copyright 1991, 1996, 1999, 2000, 2007 Free Software Foundation, Inc.
    See licenses/LICENSE-LGPLv3.txt
@@ -156,18 +154,19 @@ Applications:
 
    I2PSnark:
    Copyright (C) 2003 Mark J. Wielaard
+   GPLv2 (or any later version)
    See licenses/LICENSE-GPLv2.txt
        Silk icons: See licenses/LICENSE-SilkIcons.txt
 
    I2PTunnel:
    (c) 2003 - 2004 mihi
-   GPLv2 with exception.
+   GPLv2 (or any later version) with exception.
    See licenses/LICENSE-I2PTunnel.txt
    See licenses/LICENSE-GPLv2.txt
 
    I2PTunnel SOCKS Proxy:
    Copyright (c) 2004 by human
-   GPLv2 with exception.
+   GPLv2 (or any later version) with exception.
    See licenses/LICENSE-I2PTunnel.txt
    See licenses/LICENSE-GPLv2.txt
 
@@ -175,9 +174,9 @@ Applications:
    By welterde.
    See licenses/LICENSE-GPLv2.txt
 
-   Jetty 5.1.15:
-   Copyright 2000-2004 Mort Bay Consulting Pty. Ltd.
-   See licenses/LICENSE-Apache1.1.txt
+   Jetty 6.1.26:
+   Copyright 1995-2009 Mort Bay Consulting Pty Ltd
+   See licenses/LICENSE-Jetty.txt
    See licenses/LICENSE-Apache2.0.txt
    See licenses/NOTICE-Commons-Logging.txt
 
@@ -194,17 +193,26 @@ Applications:
 
    Router console:
    Public domain.
-       Flag icons: public domain, courtesy mjames@gmail.com http://www.famfamfam.com/
+       Flag icons:
+         - Jersey and EU flag icons: public domain, courtesy Xrmap flag
+           collection http://www.arvernes.com/wiki/index.php/Xrmap
+         - Guernsey and Isle of Man flags from the Open Clip Art Library, released into the public domain
+         - All other flag icons: public domain, courtesy mjames@gmail.com http://www.famfamfam.com/
        Silk icons: See licenses/LICENSE-SilkIcons.txt
+       FatCow icons: See licenses/LICENSE-FatCowIcons.txt
 
        GeoIP Data:
-       Copyright (c) 2003 Direct Information Pvt. Ltd. All Rights Reserved.
+       Copyright (c) 2008 MaxMind, Inc.  All Rights Reserved.
        See licenses/LICENSE-GeoIP.txt
 
    Router Console and I2PSnark themes:
    "Man with hat over face" & related images licensed under a Creative Commons 2.0 license.
    Original photos by Florian Kuhlmann. http://www.flickr.com/photos/floriankuhlmann/3117758155
 
+   I2PSnark light theme:
+   "Creative Commons Cat" licensed under a Creative Commons Attribution 3.0 Unported License.
+   Original photo by Boaz Arad. http://www.luxphile.com/2011/01/creative-commons-cat.html
+
    SAM:
    Public domain.
 
@@ -213,19 +221,27 @@ Applications:
 
    SusiDNS:
    Copyright (C) 2005 <susi23@mail.i2p>
+   GPLv2 (or any later version)
    See licenses/LICENSE-GPLv2.txt
-       Uses Apache Jakarta Standard Tag Library 1.1.2:
-       See licenses/LICENSE-Apache2.0.txt
+       Uses Glassfish Standard Tag Library (JSTL) 1.2:
+       Common Development and Distribution License (CDDL) version 1.0 + GNU General Public License (GPL) version 2
+       See https://glassfish.dev.java.net/public/CDDL+GPL.html
+       See licenses/LICENSE-GPLv2.txt
 
    SusiMail:
    Copyright (C) 2004-2005  <susi23@mail.i2p>
+   GPLv2 (or any later version)
    See licenses/LICENSE-GPLv2.txt
 
    Systray:
    Public domain.
-      Bundles systray4j code:
-      See licenses/LICENSE-GPLv2.txt
+      Bundles systray4j-2.4.1:
+      See licenses/LICENSE-LGPLv2.1.txt
 
+   Tomcat 6.0.35:
+   Copyright 1999-2011 The Apache Software Foundation
+   See licenses/LICENSE-Apache2.0.txt
+   See licenses/NOTICE-Tomcat.txt
 
 
 Other Applications and Libraries

README.txt

@@ -2,11 +2,18 @@ Prerequisites to build from source:
 	Java SDK (preferably Oracle/Sun or OpenJDK) 1.6.0 or higher
           Non-linux operating systems and JVMs: See http://trac.i2p2.de/wiki/java
 	Apache Ant 1.7.0 or higher
-	Optional, For multilanguage support: The xgettext, msgfmt, and msgmerge tools installed
+	The xgettext, msgfmt, and msgmerge tools installed
 	  from the GNU gettext package http://www.gnu.org/software/gettext/
 
 To build:
-	ant pkg
+	On x86 systems do:
+		ant pkg
+
+	On non-x86, use one of the following instead:
+		ant installer-linux
+		ant installer-freebsd
+		ant installer-osx
+
 	Run 'ant' with no arguments to see other build options.
 	See INSTALL.txt or http://www.i2p2.de/download.html for installation instructions.
 

Slackware/i2p/doinst.sh

@@ -26,9 +26,9 @@ for i in *.config ; {
 
 ( cd $INST_DIR/eepsite
 	if [ -f jetty.xml ] ; then
-		rm jetty.xml.new
+		echo "Please check ${INST_DIR}/eepsite, as there are new files."
 	else
-		mv jetty.xml.new jetty.xml
+		find $PKG/$INSTALL_DIR/i2p -name "*.xml.new" -exec sh -c 'mv "$0" "${0/.new}"' {} \;
 	fi
 )
 

Slackware/i2p/i2p.SlackBuild

@@ -18,6 +18,7 @@
 #
 
 BUILD=1sponge
+# INSTALL_DIR is referenced from /, don't prefix it with a '/'
 INSTALL_DIR=opt
 NAME=i2p
 ARCH=noarch
@@ -104,7 +105,10 @@ sed "s|%INSTALL_PATH|/$INSTALL_DIR/i2p|g" runplain.sh > a
 sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" a > runplain.sh
 # i2prouter %INSTALL_PATH and %SYSTEM_java_io_tmpdir
 sed "s|%INSTALL_PATH|/$INSTALL_DIR/i2p|g" i2prouter > a
-sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" a > i2prouter
+rm i2prouter
+mv a i2prouter
+sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" i2prouter > a
+sed "s|#ALLOW_ROOT=true|ALLOW_ROOT=true|g" a > i2prouter
 
 chmod 744 ./i2prouter
 chmod 744 ./osid
@@ -116,7 +120,7 @@ rm -Rf ./lib/*.dll ./*.bat ./*.exe ./installer ./icons ./a postinstall.sh
 
 mv $PKG/$INSTALL_DIR/i2p/*.config $PKG/install
 mv $PKG/$INSTALL_DIR/i2p/blocklist.txt $PKG/$INSTALL_DIR/i2p/blocklist.txt.new
-mv $PKG/$INSTALL_DIR/i2p/eepsite/jetty.xml $PKG/$INSTALL_DIR/i2p/eepsite/jetty.xml.new
+find $PKG/$INSTALL_DIR/i2p -name "*.xml" -exec mv {} {}.new \;
 mv $PKG/$INSTALL_DIR/i2p/eepsite/docroot/index.html $PKG/$INSTALL_DIR/i2p/eepsite/docroot/index.html.new
 mv $PKG/$INSTALL_DIR/i2p/eepsite/docroot/favicon.ico $PKG/$INSTALL_DIR/i2p/eepsite/docroot/favicon.ico.new
 sed "s|directory|/$INSTALL_DIR/i2p/|g" $CWD/doinst.sh > $PKG/install/doinst.sh

apps/BOB/src/net/i2p/BOB/BOB.java

@@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import net.i2p.I2PAppContext;
 import net.i2p.client.I2PClient;
-import net.i2p.client.streaming.RetransmissionTimer;
 import net.i2p.util.Log;
 import net.i2p.util.SimpleScheduler;
 import net.i2p.util.SimpleTimer2;
@@ -116,7 +115,6 @@ import net.i2p.util.SimpleTimer2;
  */
 public class BOB {
 
-	private final static Log _log = new Log(BOB.class);
 	public final static String PROP_CONFIG_LOCATION = "BOB.config";
 	public final static String PROP_BOB_PORT = "BOB.port";
 	public final static String PROP_BOB_HOST = "BOB.host";
@@ -138,7 +136,7 @@ public class BOB {
 	 */
 	public static void info(String arg) {
 		System.out.println("INFO:" + arg);
-		_log.info(arg);
+		(new Log(BOB.class)).info(arg);
 	}
 
 	/**
@@ -148,7 +146,7 @@ public class BOB {
 	 */
 	public static void warn(String arg) {
 		System.out.println("WARNING:" + arg);
-		_log.warn(arg);
+		(new Log(BOB.class)).warn(arg);
 	}
 
 	/**
@@ -158,7 +156,7 @@ public class BOB {
 	 */
 	public static void error(String arg) {
 		System.out.println("ERROR: " + arg);
-		_log.error(arg);
+		(new Log(BOB.class)).error(arg);
 	}
 
 	/**
@@ -182,12 +180,11 @@ public class BOB {
 		// Re-reading the config file in each thread is pretty damn stupid.
 		String configLocation = System.getProperty(PROP_CONFIG_LOCATION, "bob.config");
 		// This is here just to ensure there is no interference with our threadgroups.
-		RetransmissionTimer Y = RetransmissionTimer.getInstance();
 		SimpleScheduler Y1 = SimpleScheduler.getInstance();
 		SimpleTimer2 Y2 = SimpleTimer2.getInstance();
-		i = Y.hashCode();
 		i = Y1.hashCode();
 		i = Y2.hashCode();
+		Log _log = new Log(BOB.class);
 		try {
 			{
 				File cfg = new File(configLocation);
@@ -263,6 +260,7 @@ public class BOB {
 
 			i = 0;
 			boolean g = false;
+			spin.set(true);
 			try {
 				info("BOB is now running.");
 				listener = new ServerSocket(Integer.parseInt(props.getProperty(PROP_BOB_PORT)), 10, InetAddress.getByName(props.getProperty(PROP_BOB_HOST)));

apps/BOB/src/net/i2p/BOB/DoCMDS.java

@@ -29,6 +29,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.net.Socket;
+import java.util.Locale;
 import java.util.Properties;
 import java.util.StringTokenizer;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -53,7 +54,7 @@ public class DoCMDS implements Runnable {
 
 	// FIX ME
 	// I need a better way to do versioning, but this will do for now.
-	public static final String BMAJ = "00",  BMIN = "00",  BREV = "0F",  BEXT = "";
+	public static final String BMAJ = "00",  BMIN = "00",  BREV = "10",  BEXT = "";
 	public static final String BOBversion = BMAJ + "." + BMIN + "." + BREV + BEXT;
 	private Socket server;
 	private Properties props;
@@ -399,7 +400,7 @@ public class DoCMDS implements Runnable {
 	 */
 	private boolean is64ok(String data) {
 		try {
-			Destination x = new Destination(data);
+			new Destination(data);
 			return true;
 		} catch (Exception e) {
 			return false;
@@ -434,7 +435,7 @@ public class DoCMDS implements Runnable {
 							if (token.countTokens() != 0) {
 								Command = token.nextToken();
 								Command =
-									Command.toLowerCase();
+									Command.toLowerCase(Locale.US);
 								if (token.countTokens() != 0) {
 									Arg = token.nextToken();
 								} else {

apps/BOB/src/net/i2p/BOB/I2Plistener.java

@@ -41,7 +41,6 @@ public class I2Plistener implements Runnable {
 
 	private NamedDB info,  database;
 	private Log _log;
-//	private int tgwatch;
 	public I2PSocketManager socketManager;
 	public I2PServerSocket serverSocket;
 	private AtomicBoolean lives;
@@ -103,7 +102,7 @@ public class I2Plistener implements Runnable {
 				serverSocket.close();
 			} catch (I2PException ex) {
 			}
-		// System.out.println("I2Plistener: Close");
+			// System.out.println("I2Plistener: Close");
 		}
 	}
 }

apps/BOB/src/net/i2p/BOB/MUXlisten.java

@@ -108,16 +108,6 @@ public class MUXlisten implements Runnable {
 				this.listener = new ServerSocket(port, backlog, host);
 			}
 			socketManager = I2PSocketManagerFactory.createManager(prikey, Q);
-		// I2PException, IOException, RuntimeException
-		// To bad we can't just catch and enumerate....
-		// } catch (I2PException e) {
-		// Something went bad.
-		//	this.database.getWriteLock();
-		//	this.info.getWriteLock();
-		//	this.info.add("STARTING", new Boolean(false));
-		//	this.info.releaseWriteLock();
-		//	this.database.releaseWriteLock();
-		//	throw new I2PException(e);
 		} catch (IOException e) {
 			// Something went bad.
 			this.database.getWriteLock();
@@ -194,7 +184,6 @@ public class MUXlisten implements Runnable {
 				lock.set(false);
 				return;
 			}
-//		socketManager.addDisconnectListener(new DisconnectListener());
 			lives.set(true);
 			lock.set(false);
 			quit:
@@ -347,7 +336,6 @@ public class MUXlisten implements Runnable {
 				String boner = tg.getName();
 				// System.out.println("BOB: MUXlisten: Starting thread collection for: " + boner);
 				_log.warn("BOB: MUXlisten: Starting thread collection for: " + boner);
-				// tg.interrupt(); // give my stuff a small smack again.
 				if (tg.activeCount() + tg.activeGroupCount() != 0) {
 					// visit(tg, 0, boner);
 					int foo = tg.activeCount() + tg.activeGroupCount();

apps/BOB/src/net/i2p/BOB/Main.java

@@ -23,7 +23,6 @@
  */
 package net.i2p.BOB;
 
-import net.i2p.client.streaming.RetransmissionTimer;
 import net.i2p.util.SimpleScheduler;
 import net.i2p.util.SimpleTimer2;
 
@@ -40,7 +39,6 @@ public class Main {
 	 */
 	public static void main(String[] args) {
 		// THINK THINK THINK THINK THINK THINK
-		RetransmissionTimer Y = RetransmissionTimer.getInstance();
 		SimpleScheduler Y1 = SimpleScheduler.getInstance();
 		SimpleTimer2 Y2 = SimpleTimer2.getInstance();
 
@@ -48,6 +46,5 @@ public class Main {
 
 		Y2.stop();
 		Y1.stop();
-		Y.stop();
 	}
 }

apps/BOB/src/net/i2p/BOB/TCPio.java

@@ -95,14 +95,9 @@ public class TCPio implements Runnable {
 					if (b > 0) {
 						Aout.write(a, 0, b);
 					} else if (b == 0) {
-// Will this die? We'll see.
 						while(Ain.available() == 0) {
 							Thread.sleep(20);
 						}
-//						Thread.yield(); // this should act like a mini sleep.
-//						if (Ain.available() == 0) {
-//							Thread.sleep(10);
-//						}
 					} else {
 						/* according to the specs:
 						 *

apps/BOB/src/net/i2p/BOB/TCPlistener.java

@@ -27,8 +27,6 @@ import java.io.IOException;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
-// import net.i2p.client.I2PSession;
-// import net.i2p.client.I2PSessionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.client.streaming.I2PSocketManager;

apps/BOB/src/net/i2p/BOB/TCPtoI2P.java

@@ -30,6 +30,7 @@ import java.io.OutputStream;
 import java.net.ConnectException;
 import java.net.NoRouteToHostException;
 import java.net.Socket;
+import java.util.Locale;
 import java.util.concurrent.atomic.AtomicBoolean;
 import net.i2p.I2PException;
 import net.i2p.client.streaming.I2PSocket;
@@ -107,20 +108,18 @@ public class TCPtoI2P implements Runnable {
 	 * @throws java.io.IOException
 	 */
 	private void Emsg(String e, OutputStream out) throws IOException {
-// Debugging		System.out.println("ERROR TCPtoI2P: " + e);
+		// Debugging		System.out.println("ERROR TCPtoI2P: " + e);
 		out.write("ERROR ".concat(e).getBytes());
 		out.write(13);
 		out.write(10);
 		out.flush();
 	}
 
-//	private void rlock() throws Exception {
 	private void rlock() {
 		database.getReadLock();
 		info.getReadLock();
 	}
 
-//	private void runlock() throws Exception {
 	private void runlock() {
 		info.releaseReadLock();
 		database.releaseReadLock();
@@ -144,7 +143,7 @@ public class TCPtoI2P implements Runnable {
 				in = sock.getInputStream();
 				out = sock.getOutputStream();
 				line = lnRead(in);
-				input = line.toLowerCase();
+				input = line.toLowerCase(Locale.US);
 				Destination dest = null;
 				if (input.endsWith(".i2p")) {
 					//dest = I2PTunnel.destFromName(input);

apps/BOB/src/net/i2p/BOB/UDPIOthread.java

@@ -111,7 +111,7 @@ public class UDPIOthread implements I2PSessionListener, Runnable {
 	 * @param size
 	 */
 	public void messageAvailable(I2PSession session, int msgId, long size) {
-//		_log.debug("Message available: id = " + msgId + " size = " + size);
+		//		_log.debug("Message available: id = " + msgId + " size = " + size);
 		try {
 			byte msg[] = session.receiveMessage(msgId);
 			out.write(msg);

apps/addressbook/build.xml

@@ -55,6 +55,8 @@
 		<property name="workspace.changes" value="" />
 			<manifest>
 				<attribute name="Main-Class" value="addressbook.Daemon"/>
+				<attribute name="Implementation-Version" value="${full.version}" />
+				<attribute name="Built-By" value="${build.built-by}" />
 		                <attribute name="Build-Date" value="${build.timestamp}" />
 		                <attribute name="Base-Revision" value="${workspace.version}" />
 		                <attribute name="Workspace-Changes" value="${workspace.changes}" />
@@ -73,6 +75,8 @@
 		<property name="workspace.changes.tr" value="" />
 		<war basedir="${dist}/tmp" webxml="web.xml" destfile="${dist}/${war}">
 			<manifest>
+				<attribute name="Implementation-Version" value="${full.version}" />
+				<attribute name="Built-By" value="${build.built-by}" />
 		                <attribute name="Build-Date" value="${build.timestamp}" />
 		                <attribute name="Base-Revision" value="${workspace.version}" />
 		                <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />

apps/addressbook/java/src/net/i2p/addressbook/ConfigIterator.java

@@ -27,6 +27,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
 import java.util.NoSuchElementException;
 
@@ -71,7 +72,7 @@ class ConfigIterator implements Iterator<Map.Entry<String, String>> {
                 inputLine = ConfigParser.stripComments(inputLine);
                 String[] splitLine = inputLine.split("=");
                 if (splitLine.length == 2) {
-                    next = new ConfigEntry(splitLine[0].trim().toLowerCase(), splitLine[1].trim());
+                    next = new ConfigEntry(splitLine[0].trim().toLowerCase(Locale.US), splitLine[1].trim());
                     return true;
                 }
                 inputLine = input.readLine();

apps/addressbook/java/src/net/i2p/addressbook/ConfigParser.java

@@ -32,6 +32,7 @@ import java.io.StringReader;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import net.i2p.util.SecureFile;
@@ -92,7 +93,7 @@ class ConfigParser {
             inputLine = ConfigParser.stripComments(inputLine);
             String[] splitLine = inputLine.split("=");
             if (splitLine.length == 2) {
-                result.put(splitLine[0].trim().toLowerCase(), splitLine[1].trim());
+                result.put(splitLine[0].trim().toLowerCase(Locale.US), splitLine[1].trim());
             }
             inputLine = input.readLine();
         }

apps/addressbook/java/src/net/i2p/addressbook/Servlet.java

@@ -84,7 +84,7 @@ public class Servlet extends HttpServlet {
         this.thread.setDaemon(true);
         this.thread.setName("Addressbook");
         this.thread.start();
-        System.out.println("INFO: Starting Addressbook " + Daemon.VERSION);
+        //System.out.println("INFO: Starting Addressbook " + Daemon.VERSION);
         //System.out.println("INFO: config root under " + args[0]);
     }
 

apps/addressbook/java/src/net/i2p/addressbook/SubscriptionIterator.java

@@ -26,7 +26,6 @@ import java.util.Iterator;
 import java.util.List;
 
 import net.i2p.I2PAppContext;
-import net.i2p.data.DataHelper;  // debug
 
 /**
  * An iterator over the subscriptions in a SubscriptionList.  Note that this iterator

apps/desktopgui/locale/messages_cs.po

@@ -0,0 +1,55 @@
+# I2P
+# Copyright (C) 2009 The I2P Project
+# This file is distributed under the same license as the desktopgui package.
+# To contribute translations, see http://www.i2p2.de/newdevelopers
+# 
+# Translators:
+msgid ""
+msgstr ""
+"Project-Id-Version: I2P\n"
+"Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
+"POT-Creation-Date: 2011-03-03 18:29+0000\n"
+"PO-Revision-Date: 2012-02-12 19:44+0000\n"
+"Last-Translator: Waseihou Watashi <waseihou@gmail.com>\n"
+"Language-Team: Czech (http://www.transifex.net/projects/p/I2P/language/cs/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: cs\n"
+"Plural-Forms: nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2\n"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:23
+msgid "Start I2P"
+msgstr "Spustit I2P"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "I2P is starting!"
+msgstr "I2P startuje!"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "Starting"
+msgstr "Startuji"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:26
+msgid "Launch I2P Browser"
+msgstr "Spouštím I2P Browser"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:50
+msgid "Configure desktopgui"
+msgstr "Nastavuji desktopgui"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:67
+msgid "Restart I2P"
+msgstr "Restart I2P"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:85
+msgid "Stop I2P"
+msgstr "Zastavit I2P"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
+msgid "Tray icon configuration"
+msgstr "Nastavení ikony na liště (tray icon)"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
+msgid "Should tray icon be enabled?"
+msgstr "Zapnout ikonu na liště?"

apps/desktopgui/locale/messages_el.po

@@ -0,0 +1,56 @@
+# I2P
+# Copyright (C) 2009 The I2P Project
+# This file is distributed under the same license as the desktopgui package.
+# To contribute translations, see http://www.i2p2.de/newdevelopers
+# 
+# Translators:
+#   <lixtetrax@grhack.net>, 2012.
+msgid ""
+msgstr ""
+"Project-Id-Version: I2P\n"
+"Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
+"POT-Creation-Date: 2011-03-03 18:29+0000\n"
+"PO-Revision-Date: 2012-07-02 11:28+0000\n"
+"Last-Translator: lixtetrax <lixtetrax@grhack.net>\n"
+"Language-Team: Greek (http://www.transifex.com/projects/p/I2P/language/el/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: el\n"
+"Plural-Forms: nplurals=2; plural=(n != 1)\n"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:23
+msgid "Start I2P"
+msgstr "Έναρξη Ι2Ρ"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "I2P is starting!"
+msgstr "Το Ι2Ρ ξεκίνησε!"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "Starting"
+msgstr "Έναρξη"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:26
+msgid "Launch I2P Browser"
+msgstr "Έναρξη φυλλομετρητή Ι2Ρ"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:50
+msgid "Configure desktopgui"
+msgstr "Παραμετροποίηση desktopgui"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:67
+msgid "Restart I2P"
+msgstr "Επανεκκίνηση Ι2Ρ"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:85
+msgid "Stop I2P"
+msgstr "Τερματισμός Ι2Ρ"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
+msgid "Tray icon configuration"
+msgstr "Παραμετροποίηση εικονιδίου"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
+msgid "Should tray icon be enabled?"
+msgstr "Ενεργοποίηση εικονιδίου;"

apps/desktopgui/locale/messages_hu.po

@@ -0,0 +1,55 @@
+# I2P
+# Copyright (C) 2009 The I2P Project
+# This file is distributed under the same license as the desktopgui package.
+# To contribute translations, see http://www.i2p2.de/newdevelopers
+# 
+# Translators:
+msgid ""
+msgstr ""
+"Project-Id-Version: I2P\n"
+"Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
+"POT-Creation-Date: 2011-03-03 18:29+0000\n"
+"PO-Revision-Date: 2012-06-01 16:28+0000\n"
+"Last-Translator: AdminLMH <lehetmashogy@i2pmail.org>\n"
+"Language-Team: Hungarian (http://www.transifex.net/projects/p/I2P/language/hu/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: hu\n"
+"Plural-Forms: nplurals=2; plural=(n != 1)\n"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:23
+msgid "Start I2P"
+msgstr "I2P indítása"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "I2P is starting!"
+msgstr "I2P indul!"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "Starting"
+msgstr "indítás"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:26
+msgid "Launch I2P Browser"
+msgstr "I2P Böngésző Indítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:50
+msgid "Configure desktopgui"
+msgstr "Asztali Grafikus Felület Beállítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:67
+msgid "Restart I2P"
+msgstr "I2P Újraindítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:85
+msgid "Stop I2P"
+msgstr "I2P Leállítása"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
+msgid "Tray icon configuration"
+msgstr "Tálcaikon beállítása"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
+msgid "Should tray icon be enabled?"
+msgstr "Tálcaikon engedélyezve legyen?"

apps/desktopgui/locale/messages_it.po

@@ -3,15 +3,17 @@
 # This file is distributed under the same license as the desktopgui package.
 # To contribute translations, see http://www.i2p2.de/newdevelopers
 # 
+# Translators:
+#   <bovas85@gmail.com>, 2012.
 #   <jokjok@hotmail.it>, 2011.
 msgid ""
 msgstr ""
 "Project-Id-Version: I2P\n"
 "Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
 "POT-Creation-Date: 2011-03-03 18:29+0000\n"
-"PO-Revision-Date: 2011-06-09 17:09+0000\n"
-"Last-Translator: mkkid <jokjok@hotmail.it>\n"
-"Language-Team: Italian (http://www.transifex.net/projects/p/I2P/team/it/)\n"
+"PO-Revision-Date: 2012-06-01 12:21+0000\n"
+"Last-Translator: Leelium <bovas85@gmail.com>\n"
+"Language-Team: Italian (http://www.transifex.net/projects/p/I2P/language/it/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -48,10 +50,8 @@ msgstr "Ferma I2P"
 
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
 msgid "Tray icon configuration"
-msgstr "Configurazione dell'icona nell'Area di notifica"
+msgstr "Configurazione dell'icona nell'area di notifica"
 
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
 msgid "Should tray icon be enabled?"
-msgstr "Vuoi che l'icona nell'Area di notifica venga abilitata?"
-
-
+msgstr "Vuoi che l'icona nelll'rea di notifica venga abilitata?"

apps/desktopgui/locale/messages_sv.po

@@ -3,7 +3,8 @@
 # This file is distributed under the same license as the desktopgui package.
 # To contribute translations, see http://www.i2p2.de/newdevelopers
 # 
-# 123hund123 <M8R-ra4r1r@mailinator.com>, 2011
+# Translators:
+# 123hund123 <M8R-ra4r1r@mailinator.com>, 2011.
 msgid ""
 msgstr ""
 "Project-Id-Version: I2P\n"
@@ -11,7 +12,7 @@ msgstr ""
 "POT-Creation-Date: 2011-03-03 18:29+0000\n"
 "PO-Revision-Date: 2011-03-22 15:49+0000\n"
 "Last-Translator: 123hund123 <M8R-ra4r1r@mailinator.com>\n"
-"Language-Team: Swedish (Sweden) (http://www.transifex.net/projects/p/I2P/team/sv_SE/)\n"
+"Language-Team: Swedish (Sweden) (http://www.transifex.net/projects/p/I2P/language/sv_SE/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -53,5 +54,3 @@ msgstr "Ikonpanelskonfiguration"
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
 msgid "Should tray icon be enabled?"
 msgstr "Ska ikonpanelen vara aktiverad?"
-
-

apps/i2psnark/java/build.xml

@@ -19,6 +19,7 @@
                 <pathelement location="../../ministreaming/java/build/obj" />
                 <pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
                 <pathelement location="../../jetty/jettylib/javax.servlet.jar" />
+                <pathelement location="../../jetty/jettylib/jetty-util.jar" />
             </classpath>
         </depend>
     </target>
@@ -34,7 +35,7 @@
             debug="true" deprecation="on" source="1.5" target="1.5" 
             destdir="./build/obj" 
             includeAntRuntime="false"
-            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../ministreaming/java/build/mstreaming.jar" >
+            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jetty/jettylib/jetty-util.jar:../../ministreaming/java/build/mstreaming.jar" >
             <compilerarg line="${javac.compilerargs}" />
         </javac>
     </target>
@@ -56,10 +57,12 @@
     <target name="jar" depends="builddep, compile, jarUpToDate, listChangedFiles" unless="jar.uptodate" >
         <!-- set if unset -->
         <property name="workspace.changes.tr" value="" />
-        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/messages_*.class">
+        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class">
             <manifest>
                 <attribute name="Main-Class" value="org.klomp.snark.Snark" />
                 <attribute name="Class-Path" value="i2p.jar mstreaming.jar streaming.jar" />
+                <attribute name="Implementation-Version" value="${full.version}" />
+                <attribute name="Built-By" value="${build.built-by}" />
                 <attribute name="Build-Date" value="${build.timestamp}" />
                 <attribute name="Base-Revision" value="${workspace.version}" />
                 <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />
@@ -69,7 +72,7 @@
     
     <target name="jarUpToDate">
         <uptodate property="jar.uptodate" targetfile="build/i2psnark.jar" >
-            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/messages_*.class" />
+            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class" />
         </uptodate>
         <condition property="shouldListChanges" >
             <and>
@@ -94,10 +97,16 @@
     <target name="war" depends="jar, bundle, warUpToDate, listChangedFiles" unless="war.uptodate" > 
         <!-- set if unset -->
         <property name="workspace.changes.tr" value="" />
-        <war destfile="../i2psnark.war" webxml="../web.xml" basedir="../" includes="_icons/*" >
+        <copy todir="build/icons/.icons" >
+            <fileset dir="../icons/" />
+        </copy>
+        <war destfile="../i2psnark.war" webxml="../web.xml" >
           <!-- include only the web stuff, as of 0.7.12 the router will add i2psnark.jar to the classpath for the war -->
           <classes dir="./build/obj" includes="**/web/*.class" />
+            <fileset dir="build/icons/" />
             <manifest>
+                <attribute name="Implementation-Version" value="${full.version}" />
+                <attribute name="Built-By" value="${build.built-by}" />
                 <attribute name="Build-Date" value="${build.timestamp}" />
                 <attribute name="Base-Revision" value="${workspace.version}" />
                 <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />
@@ -107,7 +116,7 @@
     
     <target name="warUpToDate">
         <uptodate property="war.uptodate" targetfile="../i2psnark.war" >
-            <srcfiles dir= "." includes="build/obj/org/klomp/snark/web/*.class ../_icons/* ../web.xml" />
+            <srcfiles dir= "." includes="build/obj/org/klomp/snark/web/*.class ../icons/* ../web.xml" />
         </uptodate>
     </target>
     

apps/i2psnark/java/src/net/i2p/kademlia/KBucket.java

@@ -0,0 +1,76 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain 
+ * with no warranty of any kind, either expressed or implied.  
+ * It probably won't make your computer catch on fire, or eat 
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.util.Set;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Group, without inherent ordering, a set of keys a certain distance away from
+ * a local key, using XOR as the distance metric
+ *
+ * Refactored from net.i2p.router.networkdb.kademlia
+ * @since 0.9.2
+ */
+public interface KBucket<T extends SimpleDataStructure> {
+
+    /** 
+     * Lowest order high bit for difference keys.
+     * The lower-bounds distance of this bucket is 2**begin.
+     * If begin == 0, this is the closest bucket.
+     */
+    public int getRangeBegin();
+
+    /**
+     * Highest high bit for the difference keys.
+     * The upper-bounds distance of this bucket is (2**(end+1)) - 1.
+     * If begin == end, the bucket cannot be split further.
+     * If end == (numbits - 1), this is the furthest bucket.
+     */
+    public int getRangeEnd();
+
+    /**
+     * Number of keys already contained in this kbucket
+     */
+    public int getKeyCount();
+
+    /**
+     * Add the peer to the bucket
+     *
+     * @return true if added
+     */
+    public boolean add(T key);
+
+    /**
+     * Remove the key from the bucket
+     * @return true if the key existed in the bucket before removing it, else false
+     */
+    public boolean remove(T key);
+    
+    /**
+     *  Update the last-changed timestamp to now.
+     */
+    public void setLastChanged();
+
+    /**
+     *  The last-changed timestamp
+     */
+    public long getLastChanged();
+
+    /**
+     * Retrieve all routing table entries stored in the bucket
+     * @return set of Hash structures
+     */
+    public Set<T> getEntries();
+
+    public void getEntries(SelectionCollector<T> collector);
+
+    public void clear();
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketImpl.java

@@ -0,0 +1,150 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain
+ * with no warranty of any kind, either expressed or implied.
+ * It probably won't make your computer catch on fire, or eat
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.util.Collections;
+import java.util.Set;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.ConcurrentHashSet;
+
+/**
+ *  A concurrent implementation using ConcurrentHashSet.
+ *  The max size (K) may be temporarily exceeded due to concurrency,
+ *  a pending split, or the behavior of the supplied trimmer,
+ *  as explained below.
+ *  The creator is responsible for splits.
+ *
+ *  This class has no knowledge of the DHT base used for XORing,
+ *  and thus there are no validity checks in add/remove.
+ *
+ *  The begin and end values are immutable.
+ *  All entries in this bucket will have at least one bit different
+ *  from us in the range [begin, end] inclusive.
+ *  Splits must be implemented by creating two new buckets
+ *  and discarding this one.
+ *
+ *  The keys are kept in a Set and are NOT sorted by last-seen.
+ *  Per-key last-seen-time, failures, etc. must be tracked elsewhere.
+ *
+ *  If this bucket is full (i.e. begin == end && size == max)
+ *  then add() will call KBucketTrimmer.trim() do
+ *  (possibly) remove older entries, and indicate whether
+ *  to add the new entry. If the trimmer returns true without
+ *  removing entries, this KBucket will exceed the max size.
+ *
+ *  Refactored from net.i2p.router.networkdb.kademlia
+ *  @since 0.9.2
+ */
+class KBucketImpl<T extends SimpleDataStructure> implements KBucket<T> {
+    /**
+     *  set of Hash objects for the peers in the kbucket
+     */
+    private final Set<T> _entries;
+    /** include if any bits equal or higher to this bit (in big endian order) */
+    private final int _begin;
+    /** include if no bits higher than this bit (inclusive) are set */
+    private final int _end;
+    private final int _max;
+    private final KBucketTrimmer _trimmer;
+    /** when did we last shake things up */
+    private long _lastChanged;
+    private final I2PAppContext _context;
+    
+    /**
+     *  All entries in this bucket will have at least one bit different
+     *  from us in the range [begin, end] inclusive.
+     */
+    public KBucketImpl(I2PAppContext context, int begin, int end, int max, KBucketTrimmer trimmer) {
+        if (begin > end)
+            throw new IllegalArgumentException(begin + " > " + end);
+        _context = context;
+        _entries = new ConcurrentHashSet(max + 4);
+        _begin = begin;
+        _end = end;
+        _max = max;
+        _trimmer = trimmer;
+    }
+    
+    public int getRangeBegin() { return _begin; }
+
+    public int getRangeEnd() { return _end; }
+
+    public int getKeyCount() {
+        return _entries.size();
+    }
+    
+    /**
+     *  @return an unmodifiable view; not a copy
+     */
+    public Set<T> getEntries() {
+        return Collections.unmodifiableSet(_entries);
+    }
+
+    public void getEntries(SelectionCollector<T> collector) {
+        for (T h : _entries) {
+             collector.add(h);
+        }
+    }
+    
+    public void clear() {
+        _entries.clear();
+    }
+    
+    /**
+     *  Sets last-changed if rv is true OR if the peer is already present.
+     *  Calls the trimmer if begin == end and we are full.
+     *  If begin != end then add it and caller must do bucket splitting.
+     *  @return true if added
+     */
+    public boolean add(T peer) {
+        if (_begin != _end || _entries.size() < _max ||
+            _entries.contains(peer) || _trimmer.trim(this, peer)) {
+            // do this even if already contains, to call setLastChanged()
+            boolean rv = _entries.add(peer);
+            setLastChanged();
+            return rv;
+        }
+        return false;
+    }
+    
+    /**
+     *  @return if removed. Does NOT set lastChanged.
+     */
+    public boolean remove(T peer) {
+        boolean rv = _entries.remove(peer);
+        //if (rv)
+        //    setLastChanged();
+        return rv;
+    }
+    
+    /**
+     *  Update the last-changed timestamp to now.
+     */
+    public void setLastChanged() {
+        _lastChanged = _context.clock().now();
+    }
+
+    /**
+     *  The last-changed timestamp, which actually indicates last-added or last-seen.
+     */
+    public long getLastChanged() {
+        return _lastChanged;
+    }
+    
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(1024);
+        buf.append(_entries.size());
+        buf.append(" entries in (").append(_begin).append(',').append(_end);
+        buf.append(") : ").append(_entries.toString());
+        return buf.toString();
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketSet.java

@@ -0,0 +1,790 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain
+ * with no warranty of any kind, either expressed or implied.
+ * It probably won't make your computer catch on fire, or eat
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.Log;
+
+/**
+ * In-memory storage of buckets sorted by the XOR metric from the base (us)
+ * passed in via the constructor.
+ * This starts with one bucket covering the whole key space, and
+ * may eventually be split to a max of the number of bits in the data type
+ * (160 for SHA1Hash or 256 for Hash),
+ * times 2**(B-1) for Kademlia value B.
+ *
+ * Refactored from net.i2p.router.networkdb.kademlia
+ * @since 0.9.2
+ */
+public class KBucketSet<T extends SimpleDataStructure> {
+    private final Log _log;
+    private final I2PAppContext _context;
+    private final T _us;
+
+    /**
+     * The bucket list is locked by _bucketsLock, however the individual
+     * buckets are not locked. Users may see buckets that have more than
+     * the maximum k entries, or may have adds and removes silently fail
+     * when they appear to succeed.
+     *
+     * Closest values are in bucket 0, furthest are in the last bucket.
+     */
+    private final List<KBucket> _buckets;
+    private final Range<T> _rangeCalc;
+    private final KBucketTrimmer _trimmer;
+    
+    /**
+     *  Locked for reading only when traversing all the buckets.
+     *  Locked for writing only when splitting a bucket.
+     *  Adds/removes/gets from individual buckets are not locked.
+     */
+    private final ReentrantReadWriteLock _bucketsLock = new ReentrantReadWriteLock(false);
+
+    private final int KEYSIZE_BITS;
+    private final int NUM_BUCKETS;
+    private final int BUCKET_SIZE;
+    private final int B_VALUE;
+    private final int B_FACTOR;
+    
+    /**
+     * Use the default trim strategy, which removes a random entry.
+     * @param us the local identity (typically a SHA1Hash or Hash)
+     *           The class must have a zero-argument constructor.
+     * @param max the Kademlia value "k", the max per bucket, k >= 4
+     * @param b the Kademlia value "b", split buckets an extra 2**(b-1) times,
+     *           b > 0, use 1 for bittorrent, Kademlia paper recommends 5
+     */
+    public KBucketSet(I2PAppContext context, T us, int max, int b) {
+        this(context, us, max, b, new RandomTrimmer(context, max));
+    }
+
+    /**
+     * Use the supplied trim strategy.
+     */
+    public KBucketSet(I2PAppContext context, T us, int max, int b, KBucketTrimmer trimmer) {
+        _us = us;
+        _context = context;
+        _log = context.logManager().getLog(KBucketSet.class);
+        _trimmer = trimmer;
+        if (max <= 4 || b <= 0 || b > 8)
+            throw new IllegalArgumentException();
+        KEYSIZE_BITS = us.length() * 8;
+        B_VALUE = b;
+        B_FACTOR = 1 << (b - 1);
+        NUM_BUCKETS = KEYSIZE_BITS * B_FACTOR;
+        BUCKET_SIZE = max;
+        _buckets = createBuckets();
+        _rangeCalc = new Range(us, B_VALUE);
+        // this verifies the zero-argument constructor
+        makeKey(new byte[us.length()]);
+    }
+    
+    private void getReadLock() {
+        _bucketsLock.readLock().lock();
+    }
+
+    /**
+     *  Get the lock if we can. Non-blocking.
+     *  @return true if the lock was acquired
+     */
+    private boolean tryReadLock() {
+        return _bucketsLock.readLock().tryLock();
+    }
+
+    private void releaseReadLock() {
+        _bucketsLock.readLock().unlock();
+    }
+
+    /** @return true if the lock was acquired */
+    private boolean getWriteLock() {
+        try {
+            boolean rv = _bucketsLock.writeLock().tryLock(3000, TimeUnit.MILLISECONDS);
+            if ((!rv) && _log.shouldLog(Log.WARN))
+                _log.warn("no lock, size is: " + _bucketsLock.getQueueLength(), new Exception("rats"));
+            return rv;
+        } catch (InterruptedException ie) {}
+        return false;
+    }
+
+    private void releaseWriteLock() {
+        _bucketsLock.writeLock().unlock();
+    }
+
+    /**
+     * @return true if the peer is new to the bucket it goes in, or false if it was
+     *  already in it. Always returns false on an attempt to add ourselves.
+     *
+     */
+    public boolean add(T peer) {
+        KBucket bucket;
+        getReadLock();
+        try {
+            bucket = getBucket(peer);
+        } finally { releaseReadLock(); }
+        if (bucket != null) {
+            if (bucket.add(peer)) {
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Peer " + peer + " added to bucket " + bucket);
+                if (shouldSplit(bucket)) {
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Splitting bucket " + bucket);
+                    split(bucket.getRangeBegin());
+                    //testAudit(this, _log);
+                }
+                return true;
+            } else {
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Peer " + peer + " NOT added to bucket " + bucket);
+                return false;
+            }
+        } else {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Failed to add, probably us: " + peer);
+            return false;
+        }
+    }
+
+    /**
+     *  No lock required.
+     *  FIXME will split the closest buckets too far if B > 1 and K < 2**B
+     *  Won't ever really happen and if it does it still works.
+     */
+    private boolean shouldSplit(KBucket b) {
+        return
+               b.getRangeBegin() != b.getRangeEnd() &&
+               b.getKeyCount() > BUCKET_SIZE;
+    }
+
+    /**
+     *  Grabs the write lock.
+     *  Caller must NOT have the read lock.
+     *  The bucket should be splittable (range start != range end).
+     *  @param r the range start of the bucket to be split
+     */
+    private void split(int r) {
+        if (!getWriteLock())
+            return;
+        try {
+            locked_split(r);
+        } finally { releaseWriteLock(); }
+    }
+
+    /**
+     *  Creates two or more new buckets. The old bucket is replaced and discarded.
+     *
+     *  Caller must hold write lock
+     *  The bucket should be splittable (range start != range end).
+     *  @param r the range start of the bucket to be split
+     */
+    private void locked_split(int r) {
+        int b = pickBucket(r);
+        while (shouldSplit(_buckets.get(b))) {
+            KBucket<T> b0 = _buckets.get(b);
+            // Each bucket gets half the keyspace.
+            // When B_VALUE = 1, or the bucket is larger than B_FACTOR, then
+            // e.g. 0-159 => 0-158, 159-159
+            // When B_VALUE > 1, and the bucket is smaller than B_FACTOR, then
+            // e.g. 1020-1023 => 1020-1021, 1022-1023
+            int s1, e1, s2, e2;
+            s1 = b0.getRangeBegin();
+            e2 = b0.getRangeEnd();
+            if (B_FACTOR > 1 &&
+                (s1 & (B_FACTOR - 1)) == 0 &&
+                ((e2 + 1) & (B_FACTOR - 1)) == 0 &&
+                e2 > s1 + B_FACTOR) {
+                // The bucket is a "whole" kbucket with a range > B_FACTOR,
+                // so it should be split into two "whole" kbuckets each with
+                // a range >= B_FACTOR.
+                // Log split
+                s2 = e2 + 1 - B_FACTOR;
+            } else {
+                // The bucket is the smallest "whole" kbucket with a range == B_FACTOR,
+                // or B_VALUE > 1 and the bucket has already been split.
+                // Start or continue splitting down to a depth B_VALUE.
+                // Linear split
+                s2 = s1 + ((1 + e2 - s1) / 2);
+            }	
+            e1 = s2 - 1;
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Splitting (" + s1 + ',' + e2 + ") -> (" + s1 + ',' + e1 + ") (" + s2 + ',' + e2 + ')');
+            KBucket<T> b1 = createBucket(s1, e1);
+            KBucket<T> b2 = createBucket(s2, e2);
+            for (T key : b0.getEntries()) {
+                if (getRange(key) < s2)
+                    b1.add(key);
+                else
+                    b2.add(key);
+            }
+            _buckets.set(b, b1);
+            _buckets.add(b + 1, b2);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Split bucket at idx " + b +
+                           ":\n" + b0 +
+                           "\ninto: " + b1 +
+                           "\nand: " + b2);
+            //if (_log.shouldLog(Log.DEBUG))
+            //    _log.debug("State is now: " + toString());
+
+            if (b2.getKeyCount() > BUCKET_SIZE) {
+                // should be rare... too hard to call _trimmer from here
+                // (and definitely not from inside the write lock)
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("All went into 2nd bucket after split");
+            }
+            // loop if all the entries went in the first bucket
+        }
+    }
+
+    /**
+     *  The current number of entries.
+     */
+    public int size() {
+        int rv = 0;
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                rv += b.getKeyCount();
+            }
+        } finally { releaseReadLock(); }
+        return rv;
+    }
+    
+    public boolean remove(T entry) {
+        KBucket kbucket;
+        getReadLock();
+        try {
+            kbucket = getBucket(entry);
+        } finally { releaseReadLock(); }
+        boolean removed = kbucket.remove(entry);
+        return removed;
+    }
+    
+    /** @since 0.8.8 */
+    public void clear() {
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                b.clear();
+            }
+        } finally { releaseReadLock(); }
+        _rangeCalc.clear();
+    }
+    
+    /**
+     *  @return a copy in a new set
+     */
+    public Set<T> getAll() {
+        Set<T> all = new HashSet(256);
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                all.addAll(b.getEntries());
+            }
+        } finally { releaseReadLock(); }
+        return all;
+    }
+
+    /**
+     *  @return a copy in a new set
+     */
+    public Set<T> getAll(Set<T> toIgnore) {
+        Set<T> all = getAll();
+        all.removeAll(toIgnore);
+        return all;
+    }
+    
+    public void getAll(SelectionCollector<T> collector) {
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                b.getEntries(collector);
+            }
+        } finally { releaseReadLock(); }
+    }
+    
+    /**
+     *  The keys closest to us.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(int max) {
+        return getClosest(max, Collections.EMPTY_SET);
+    }
+    
+    /**
+     *  The keys closest to us.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(int max, Collection<T> toIgnore) {
+        List<T> rv = new ArrayList(max);
+        int count = 0;
+        getReadLock();
+        try {
+            // start at first (closest) bucket
+            for (int i = 0; i < _buckets.size() && count < max; i++) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                // add the whole bucket except for ignores,
+                // extras will be trimmed after sorting
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+        } finally { releaseReadLock(); }
+        Comparator comp = new XORComparator(_us);
+        Collections.sort(rv, comp);
+        int sz = rv.size();
+        for (int i = sz - 1; i >= max; i--) {
+            rv.remove(i);
+        }
+        return rv;
+    }
+    
+    /**
+     *  The keys closest to the key.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(T key, int max) {
+        return getClosest(key, max, Collections.EMPTY_SET);
+    }
+    
+    /**
+     *  The keys closest to the key.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(T key, int max, Collection<T> toIgnore) {
+        if (key.equals(_us))
+            return getClosest(max, toIgnore);
+        List<T> rv = new ArrayList(max);
+        int count = 0;
+        getReadLock();
+        try {
+            int start = pickBucket(key);
+            // start at closest bucket, then to the smaller (closer to us) buckets
+            for (int i = start; i >= 0 && count < max; i--) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+            // then the farther from us buckets if necessary
+            for (int i = start + 1; i < _buckets.size() && count < max; i++) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+        } finally { releaseReadLock(); }
+        Comparator comp = new XORComparator(key);
+        Collections.sort(rv, comp);
+        int sz = rv.size();
+        for (int i = sz - 1; i >= max; i--) {
+            rv.remove(i);
+        }
+        return rv;
+    }
+
+    /**
+     *  The bucket number (NOT the range number) that the xor of the key goes in
+     *  Caller must hold read lock
+     *  @return 0 to max-1 or -1 for us
+     */
+    private int pickBucket(T key) {
+        int range = getRange(key);
+        if (range < 0)
+            return -1;
+        int rv = pickBucket(range);
+        if (rv >= 0) {
+             return rv;
+        }
+        _log.error("Key does not fit in any bucket?! WTF!\nKey  : [" 
+                   + DataHelper.toHexString(key.getData()) + "]" 
+                   + "\nUs   : " + _us
+                   + "\nDelta: ["
+                   + DataHelper.toHexString(DataHelper.xor(_us.getData(), key.getData()))
+                   + "]", new Exception("WTF"));
+        _log.error(toString());
+        throw new IllegalStateException("pickBucket returned " + rv);
+        //return -1;
+    }
+    
+    /**
+     *  Returned list is a copy of the bucket list, closest first,
+     *  with the actual buckets (not a copy).
+     *
+     *  Primarily for testing. You shouldn't ever need to get all the buckets.
+     *  Use getClosest() or getAll() instead to get the keys.
+     *
+     *  @return non-null
+     */
+    List<KBucket<T>> getBuckets() {
+        getReadLock();
+        try {
+            return new ArrayList(_buckets);
+        } finally { releaseReadLock(); }
+    }
+
+    /**
+     *  The bucket that the xor of the key goes in
+     *  Caller must hold read lock
+     *  @return null if key is us
+     */
+    private KBucket getBucket(T key) {
+       int bucket = pickBucket(key);
+       if (bucket < 0)
+           return null;
+       return _buckets.get(bucket);
+    }
+    
+    /**
+     *  The bucket number that contains this range number
+     *  Caller must hold read lock or write lock
+     *  @return 0 to max-1 or -1 for us
+     */
+    private int pickBucket(int range) {
+        // If B is small, a linear search from back to front
+        // is most efficient since most of the keys are at the end...
+        // If B is larger, there's a lot of sub-buckets
+        // of equal size to be checked so a binary search is better
+        if (B_VALUE <= 3) {
+            for (int i = _buckets.size() - 1; i >= 0; i--) {
+                KBucket b = _buckets.get(i);
+                if (range >= b.getRangeBegin() && range <= b.getRangeEnd())
+                    return i;
+            }
+            return -1;
+        } else {
+            KBucket dummy = new DummyBucket(range);
+            return Collections.binarySearch(_buckets, dummy, new BucketComparator());
+        }
+    }
+
+    private List<KBucket> createBuckets() {
+        // just an initial size
+        List<KBucket> buckets = new ArrayList(4 * B_FACTOR);
+        buckets.add(createBucket(0, NUM_BUCKETS -1));
+        return buckets;
+    }
+    
+    private KBucket createBucket(int start, int end) {
+        if (end - start >= B_FACTOR &&
+            (((end + 1) & B_FACTOR - 1) != 0 ||
+             (start & B_FACTOR - 1) != 0))
+            throw new IllegalArgumentException("Sub-bkt crosses K-bkt boundary: " + start + '-' + end);
+        KBucket bucket = new KBucketImpl(_context, start, end, BUCKET_SIZE, _trimmer);
+        return bucket;
+    }
+    
+    /**
+     *  The number of bits minus 1 (range number) for the xor of the key.
+     *  Package private for testing only. Others shouldn't need this.
+     *  @return 0 to max-1 or -1 for us
+     */
+    int getRange(T key) {
+        return _rangeCalc.getRange(key);
+    }
+    
+    /**
+     *  For every bucket that hasn't been updated in this long,
+     *  generate a random key that would be a member of that bucket.
+     *  The returned keys may be searched for to "refresh" the buckets.
+     *  @return non-null, closest first
+     */
+    public List<T> getExploreKeys(long age) {
+        List<T> rv = new ArrayList(_buckets.size());
+        long old = _context.clock().now() - age;
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                if (b.getLastChanged() < old)
+                    rv.add(generateRandomKey(b));
+            }
+        } finally { releaseReadLock(); }
+        return rv;
+    }
+    
+    /**
+     *  Generate a random key to go within this bucket
+     *  Package private for testing only. Others shouldn't need this.
+     */
+    T generateRandomKey(KBucket bucket) {
+        int begin = bucket.getRangeBegin();
+        int end = bucket.getRangeEnd();
+        // number of fixed bits, out of B_VALUE - 1 bits
+        int fixed = 0;
+        int bsz = 1 + end - begin;
+        // compute fixed = B_VALUE - log2(bsz)
+        // e.g for B=4, B_FACTOR=8, sz 4-> fixed 1, sz 2->fixed 2, sz 1 -> fixed 3
+        while (bsz < B_FACTOR) {
+            fixed++;
+            bsz <<= 1;
+        }
+        int fixedBits = 0;
+        if (fixed > 0) {
+            // 0x01, 03, 07, 0f, ...
+            int mask = (1 << fixed) - 1;
+            // fixed bits masked from begin
+            fixedBits = (begin >> (B_VALUE - (fixed + 1))) & mask;
+        }
+        int obegin = begin;
+        int oend = end;
+        begin >>= (B_VALUE - 1);
+        end >>= (B_VALUE - 1);
+        // we need randomness for [0, begin) bits
+        BigInteger variance;
+        // 00000000rrrr
+        if (begin > 0)
+            variance = new BigInteger(begin - fixed, _context.random());
+        else
+            variance = BigInteger.ZERO;
+        // we need nonzero randomness for [begin, end] bits
+        int numNonZero = 1 + end - begin;
+        if (numNonZero == 1) {
+            // 00001000rrrr
+            variance = variance.setBit(begin);
+            // fixed bits as the 'main' bucket is split
+            // 00001fffrrrr
+            if (fixed > 0)
+                variance = variance.or(BigInteger.valueOf(fixedBits).shiftLeft(begin - fixed));
+        } else {
+            // dont span main bucket boundaries with depth > 1
+            if (fixed > 0)
+                throw new IllegalStateException("WTF " + bucket);
+            BigInteger nonz;
+            if (numNonZero <= 62) {
+                // add one to ensure nonzero
+                long nz = 1 + _context.random().nextLong((1l << numNonZero) - 1);
+                nonz = BigInteger.valueOf(nz);
+            } else {
+                // loop to ensure nonzero
+                do {
+                    nonz = new BigInteger(numNonZero, _context.random());
+                } while (nonz.equals(BigInteger.ZERO));
+            }
+            // shift left and or-in the nonzero randomness
+            if (begin > 0)
+                nonz = nonz.shiftLeft(begin);
+            // 0000nnnnrrrr
+            variance = variance.or(nonz);
+        }
+
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("SB(" + obegin + ',' + oend + ") KB(" + begin + ',' + end + ") fixed=" + fixed + " fixedBits=" + fixedBits + " numNonZ=" + numNonZero);
+        byte data[] = variance.toByteArray();
+        T key = makeKey(data);
+        byte[] hash = DataHelper.xor(key.getData(), _us.getData());
+        T rv = makeKey(hash);
+
+        // DEBUG
+        //int range = getRange(rv);
+        //if (range < obegin || range > oend) {
+        //    throw new IllegalStateException("Generate random key failed range=" + range + " for " + rv + " meant for bucket " + bucket);
+        //}
+
+        return rv;
+    }
+    
+    /**
+     *  Make a new SimpleDataStrucure from the data
+     *  @param data size <= SDS length, else throws IAE
+     *              Can be 1 bigger if top byte is zero
+     */
+    private T makeKey(byte[] data) {
+        int len = _us.length();
+        int dlen = data.length;
+        if (dlen > len + 1 ||
+            (dlen == len + 1 && data[0] != 0))
+            throw new IllegalArgumentException("bad length " + dlen + " > " + len);
+        T rv;
+        try {
+            rv = (T) _us.getClass().newInstance();
+        } catch (Exception e) {
+            _log.error("fail", e);
+            throw new RuntimeException(e);
+        }
+        if (dlen == len) {
+            rv.setData(data);
+        } else {
+            byte[] ndata = new byte[len];
+            if (dlen == len + 1) {
+                // one bigger
+                System.arraycopy(data, 1, ndata, 0, len);
+            } else {
+                // smaller
+                System.arraycopy(data, 0, ndata, len - dlen, dlen);
+            }
+            rv.setData(ndata);
+        }
+        return rv;
+    }
+
+    private static class Range<T extends SimpleDataStructure> {
+        private final int _bValue;
+        private final BigInteger _bigUs;
+        private final Map<T, Integer> _distanceCache;
+
+        public Range(T us, int bValue) {
+            _bValue = bValue;
+            _bigUs = new BigInteger(1, us.getData());
+            _distanceCache = new LHM(256);
+        }
+
+        /** @return 0 to max-1 or -1 for us */
+        public int getRange(T key) {
+            Integer rv;
+            synchronized (_distanceCache) {
+                rv = _distanceCache.get(key);
+                if (rv == null) {
+                    // easy way when _bValue == 1
+                    //rv = Integer.valueOf(_bigUs.xor(new BigInteger(1, key.getData())).bitLength() - 1);
+                    BigInteger xor = _bigUs.xor(new BigInteger(1, key.getData()));
+                    int range = xor.bitLength() - 1;
+                    if (_bValue > 1) {
+                        int toShift = range + 1 - _bValue;
+                        int highbit = range;
+                        range <<= _bValue - 1;
+                        if (toShift >= 0) {
+                            int extra = xor.clearBit(highbit).shiftRight(toShift).intValue();
+                            range += extra;
+                            //Log log = I2PAppContext.getGlobalContext().logManager().getLog(KBucketSet.class);
+                            //if (log.shouldLog(Log.DEBUG))
+                            //    log.debug("highbit " + highbit + " toshift " + toShift + " extra " + extra + " new " + range);
+                        }
+                    }
+                    rv = Integer.valueOf(range);
+                    _distanceCache.put(key, rv);
+                }
+            }
+            return rv.intValue();
+        }
+
+        public void clear() {
+            synchronized (_distanceCache) {
+                _distanceCache.clear();
+            }
+        }
+    }
+
+    private static class LHM<K, V> extends LinkedHashMap<K, V> {
+        private final int _max;
+
+        public LHM(int max) {
+            super(max, 0.75f, true);
+            _max = max;
+        }
+
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+            return size() > _max;
+        }
+    }
+
+    /**
+     *  For Collections.binarySearch.
+     *  getRangeBegin == getRangeEnd.
+     */
+    private static class DummyBucket<T extends SimpleDataStructure> implements KBucket<T> {
+        private final int r;
+
+        public DummyBucket(int range) {
+            r = range;
+        }
+
+        public int getRangeBegin() { return r; }
+        public int getRangeEnd() { return r; }
+
+        public int getKeyCount() {
+            return 0;
+        }
+
+        public Set<T> getEntries() {
+            throw new UnsupportedOperationException();
+        }
+
+        public void getEntries(SelectionCollector<T> collector) {
+            throw new UnsupportedOperationException();
+        }
+    
+        public void clear() {}
+
+        public boolean add(T peer) {
+            throw new UnsupportedOperationException();
+        }
+    
+        public boolean remove(T peer) {
+            return false;
+        }
+
+        public void setLastChanged() {}
+
+        public long getLastChanged() {
+            return 0;
+        }
+    }
+
+    /**
+     *  For Collections.binarySearch.
+     *  Returns equal for any overlap.
+     */
+    private static class BucketComparator implements Comparator<KBucket> {
+        public int compare(KBucket l, KBucket r) {
+            if (l.getRangeEnd() < r.getRangeBegin())
+                return -1;
+            if (l.getRangeBegin() > r.getRangeEnd())
+                return 1;
+            return 0;
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(1024);
+        buf.append("Bucket set rooted on: ").append(_us.toString())
+           .append(" K= ").append(BUCKET_SIZE)
+           .append(" B= ").append(B_VALUE)
+           .append(" with ").append(size())
+           .append(" keys in ").append(_buckets.size()).append(" buckets:\n");
+        getReadLock();
+        try {
+            int len = _buckets.size();
+            for (int i = 0; i < len; i++) {
+                KBucket b = _buckets.get(i);
+                buf.append("* Bucket ").append(i).append("/").append(len).append(": ");
+                buf.append(b.toString()).append("\n");
+            }
+        } finally { releaseReadLock(); }
+        return buf.toString();
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketTrimmer.java

@@ -0,0 +1,20 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Called when a kbucket can no longer be split and is too big
+ *  @since 0.9.2
+ */
+public interface KBucketTrimmer<K extends SimpleDataStructure> {
+    /**
+     *  Called from add() just before adding the entry.
+     *  You may call getEntries() and/or remove() from here.
+     *  Do NOT call add().
+     *  To always discard a newer entry, always return false.
+     *
+     *  @param kbucket the kbucket that is now too big
+     *  @return true to actually add the entry.
+     */
+    public boolean trim(KBucket<K> kbucket, K toAdd);
+}

apps/i2psnark/java/src/net/i2p/kademlia/RandomIfOldTrimmer.java

@@ -0,0 +1,21 @@
+package net.i2p.kademlia;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes a random element, but only if the bucket hasn't changed in 5 minutes.
+ *  @since 0.9.2
+ */
+public class RandomIfOldTrimmer<T extends SimpleDataStructure> extends RandomTrimmer<T> {
+
+    public RandomIfOldTrimmer(I2PAppContext ctx, int max) {
+        super(ctx, max);
+    }
+
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        if (kbucket.getLastChanged() > _ctx.clock().now() - 5*60*1000)
+            return false;
+        return super.trim(kbucket, toAdd);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/RandomTrimmer.java

@@ -0,0 +1,31 @@
+package net.i2p.kademlia;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes a random element. Not resistant to flooding.
+ *  @since 0.9.2
+ */
+public class RandomTrimmer<T extends SimpleDataStructure> implements KBucketTrimmer<T> {
+    protected final I2PAppContext _ctx;
+    private final int _max;
+
+    public RandomTrimmer(I2PAppContext ctx, int max) {
+        _ctx = ctx;
+        _max = max;
+    }
+
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        List<T> e = new ArrayList(kbucket.getEntries());
+        int sz = e.size();
+        // concurrency
+        if (sz < _max)
+            return true;
+        T toRemove = e.get(_ctx.random().nextInt(sz));
+        return kbucket.remove(toRemove);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/RejectTrimmer.java

@@ -0,0 +1,13 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes nothing and always rejects the add. Flood resistant..
+ *  @since 0.9.2
+ */
+public class RejectTrimmer<T extends SimpleDataStructure> implements KBucketTrimmer<T> {
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        return false;
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/SelectionCollector.java

@@ -0,0 +1,11 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Visit kbuckets, gathering matches
+ * @since 0.9.2
+ */
+public interface SelectionCollector<T extends SimpleDataStructure> {
+    public void add(T entry);
+}

apps/i2psnark/java/src/net/i2p/kademlia/XORComparator.java

@@ -0,0 +1,28 @@
+package net.i2p.kademlia;
+
+import java.util.Comparator;
+
+import net.i2p.data.DataHelper;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Help sort Hashes in relation to a base key using the XOR metric
+ *
+ * @since 0.9.2
+ */
+class XORComparator<T extends SimpleDataStructure> implements Comparator<T> {
+    private final byte[] _base;
+
+    /**
+     * @param target key to compare distances with
+     */
+    public XORComparator(T target) {
+        _base = target.getData();
+    }
+
+    public int compare(T lhs, T rhs) {
+        byte lhsDelta[] = DataHelper.xor(lhs.getData(), _base);
+        byte rhsDelta[] = DataHelper.xor(rhs.getData(), _base);
+        return DataHelper.compareTo(lhsDelta, rhsDelta);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/package.html

@@ -0,0 +1,6 @@
+<html><body><p>
+This is a major rewrite of KBucket, KBucketSet, and KBucketImpl from net.i2p.router.networkdb.kademlia.
+The classes are now generic to support SHA1. SHA256, or other key lengths.
+The long-term goal is to prove out this new implementation in i2psnark,
+then move it to core, then convert the network database to use it.
+</p></body></html>

apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java

@@ -29,8 +29,12 @@ import net.i2p.I2PAppContext;
 import net.i2p.I2PException;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.client.streaming.I2PSocket;
+import net.i2p.data.Hash;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
+import net.i2p.util.ObjectCounter;
+import net.i2p.util.SimpleScheduler;
+import net.i2p.util.SimpleTimer;
 
 /**
  * Accepts connections on a TCP port and routes them to sub-acceptors.
@@ -41,11 +45,15 @@ public class ConnectionAcceptor implements Runnable
   private I2PServerSocket serverSocket;
   private PeerAcceptor peeracceptor;
   private Thread thread;
-  private I2PSnarkUtil _util;
+  private final I2PSnarkUtil _util;
+  private final ObjectCounter<Hash> _badCounter = new ObjectCounter();
 
   private boolean stop;
   private boolean socketChanged;
 
+  private static final int MAX_BAD = 2;
+  private static final long BAD_CLEAN_INTERVAL = 30*60*1000;
+
   public ConnectionAcceptor(I2PSnarkUtil util) { _util = util; }
   
   public synchronized void startAccepting(PeerCoordinatorSet set, I2PServerSocket socket) {
@@ -59,6 +67,7 @@ public class ConnectionAcceptor implements Runnable
           thread = new I2PAppThread(this, "I2PSnark acceptor");
           thread.setDaemon(true);
           thread.start();
+          _util.getContext().simpleScheduler().addPeriodicEvent(new Cleaner(), BAD_CLEAN_INTERVAL);
       }
     }
   }
@@ -70,11 +79,10 @@ public class ConnectionAcceptor implements Runnable
     this.peeracceptor = peeracceptor;
     _util = util;
     
-    socketChanged = false;
-    stop = false;
     thread = new I2PAppThread(this, "I2PSnark acceptor");
     thread.setDaemon(true);
     thread.start();
+    _util.getContext().simpleScheduler().addPeriodicEvent(new Cleaner(), BAD_CLEAN_INTERVAL);
   }
 
   public void halt()
@@ -138,7 +146,13 @@ public class ConnectionAcceptor implements Runnable
                 }
             } else {
                 if (socket.getPeerDestination().equals(_util.getMyDestination())) {
-                    _util.debug("Incoming connection from myself", Snark.ERROR);
+                    _log.error("Incoming connection from myself");
+                    try { socket.close(); } catch (IOException ioe) {}
+                    continue;
+                }
+                if (_badCounter.count(socket.getPeerDestination().calculateHash()) >= MAX_BAD) {
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn("Rejecting connection from " + socket.getPeerDestination().calculateHash() + " after " + MAX_BAD + " failures");
                     try { socket.close(); } catch (IOException ioe) {}
                     continue;
                 }
@@ -149,13 +163,13 @@ public class ConnectionAcceptor implements Runnable
         catch (I2PException ioe)
           {
             if (!socketChanged) {
-                _util.debug("Error while accepting: " + ioe, Snark.ERROR);
+                _log.error("Error while accepting", ioe);
                 stop = true;
             }
           }
         catch (IOException ioe)
           {
-            _util.debug("Error while accepting: " + ioe, Snark.ERROR);
+            _log.error("Error while accepting", ioe);
             stop = true;
           }
         // catch oom?
@@ -171,10 +185,12 @@ public class ConnectionAcceptor implements Runnable
   }
   
   private class Handler implements Runnable {
-      private I2PSocket _socket;
+      private final I2PSocket _socket;
+
       public Handler(I2PSocket socket) {
           _socket = socket;
       }
+
       public void run() {
           try {
               InputStream in = _socket.getInputStream();
@@ -182,13 +198,23 @@ public class ConnectionAcceptor implements Runnable
               // this is for the readahead in PeerAcceptor.connection()
               in = new BufferedInputStream(in);
               if (_log.shouldLog(Log.DEBUG))
-                  _log.debug("Handling socket from " + _socket.getPeerDestination().calculateHash().toBase64());
+                  _log.debug("Handling socket from " + _socket.getPeerDestination().calculateHash());
               peeracceptor.connection(_socket, in, out);
+          } catch (PeerAcceptor.ProtocolException ihe) {
+              _badCounter.increment(_socket.getPeerDestination().calculateHash());
+              if (_log.shouldLog(Log.INFO))
+                  _log.info("Protocol error from " + _socket.getPeerDestination().calculateHash(), ihe);
+              try { _socket.close(); } catch (IOException ignored) { }
           } catch (IOException ioe) {
               if (_log.shouldLog(Log.DEBUG))
-                  _log.debug("Error handling connection from " + _socket.getPeerDestination().calculateHash().toBase64(), ioe);
+                  _log.debug("Error handling connection from " + _socket.getPeerDestination().calculateHash(), ioe);
               try { _socket.close(); } catch (IOException ignored) { }
           }
       }
   }
+
+    /** @since 0.9.1 */    
+    private class Cleaner implements SimpleTimer.TimedEvent {
+        public void timeReached() { _badCounter.clear(); }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java

@@ -24,7 +24,7 @@ package org.klomp.snark;
 /**
  * Callback used when some peer changes state.
  */
-public interface CoordinatorListener
+interface CoordinatorListener
 {
   /**
    * Called when the PeerCoordinator notices a change in the state of a peer.
@@ -40,4 +40,5 @@ public interface CoordinatorListener
   public boolean overUploadLimit(int uploaders);
   public boolean overUpBWLimit();
   public boolean overUpBWLimit(long total);
+  public void addMessage(String message);
 }

apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java

@@ -1,7 +1,6 @@
 package org.klomp.snark;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -15,7 +14,6 @@ import net.i2p.util.Log;
 import org.klomp.snark.bencode.BDecoder;
 import org.klomp.snark.bencode.BEncoder;
 import org.klomp.snark.bencode.BEValue;
-import org.klomp.snark.bencode.InvalidBEncodingException;
 
 /**
  * REF: BEP 10 Extension Protocol
@@ -24,14 +22,15 @@ import org.klomp.snark.bencode.InvalidBEncodingException;
  */
 abstract class ExtensionHandler {
 
-    private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class);
-
     public static final int ID_HANDSHAKE = 0;
     public static final int ID_METADATA = 1;
     public static final String TYPE_METADATA = "ut_metadata";
     public static final int ID_PEX = 2;
     /** not ut_pex since the compact format is different */
     public static final String TYPE_PEX = "i2p_pex";
+    public static final int ID_DHT = 3;
+    /** not using the option bit since the compact format is different */
+    public static final String TYPE_DHT = "i2p_dht";
     /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */
     private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4;
     private static final int PARALLEL_REQUESTS = 3;
@@ -39,15 +38,23 @@ abstract class ExtensionHandler {
 
   /**
    *  @param metasize -1 if unknown
+   *  @param pexAndMetadata advertise these capabilities
+   *  @param dht advertise DHT capability
    *  @return bencoded outgoing handshake message
    */
-    public static byte[] getHandshake(int metasize) {
+    public static byte[] getHandshake(int metasize, boolean pexAndMetadata, boolean dht) {
         Map<String, Object> handshake = new HashMap();
         Map<String, Integer> m = new HashMap();
-        m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA));
-        m.put(TYPE_PEX, Integer.valueOf(ID_PEX));
-        if (metasize >= 0)
-            handshake.put("metadata_size", Integer.valueOf(metasize));
+        if (pexAndMetadata) {
+            m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA));
+            m.put(TYPE_PEX, Integer.valueOf(ID_PEX));
+            if (metasize >= 0)
+                handshake.put("metadata_size", Integer.valueOf(metasize));
+        }
+        if (dht) {
+            m.put(TYPE_DHT, Integer.valueOf(ID_DHT));
+        }
+        // include the map even if empty so the far-end doesn't NPE
         handshake.put("m", m);
         handshake.put("p", Integer.valueOf(6881));
         handshake.put("v", "I2PSnark");
@@ -56,21 +63,24 @@ abstract class ExtensionHandler {
     }
 
     public static void handleMessage(Peer peer, PeerListener listener, int id, byte[] bs) {
-        if (_log.shouldLog(Log.INFO))
-            _log.info("Got extension msg " + id + " length " + bs.length + " from " + peer);
+        Log log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class);
+        if (log.shouldLog(Log.INFO))
+            log.info("Got extension msg " + id + " length " + bs.length + " from " + peer);
         if (id == ID_HANDSHAKE)
-            handleHandshake(peer, listener, bs);
+            handleHandshake(peer, listener, bs, log);
         else if (id == ID_METADATA)
-            handleMetadata(peer, listener, bs);
+            handleMetadata(peer, listener, bs, log);
         else if (id == ID_PEX)
-            handlePEX(peer, listener, bs);
-        else if (_log.shouldLog(Log.INFO))
-            _log.info("Unknown extension msg " + id + " from " + peer);
+            handlePEX(peer, listener, bs, log);
+        else if (id == ID_DHT)
+            handleDHT(peer, listener, bs, log);
+        else if (log.shouldLog(Log.INFO))
+            log.info("Unknown extension msg " + id + " from " + peer);
     }
 
-    private static void handleHandshake(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got handshake msg from " + peer);
+    private static void handleHandshake(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got handshake msg from " + peer);
         try {
             // this throws NPE on missing keys
             InputStream is = new ByteArrayInputStream(bs);
@@ -81,20 +91,26 @@ abstract class ExtensionHandler {
             Map<String, BEValue> msgmap = map.get("m").getMap();
 
             if (msgmap.get(TYPE_PEX) != null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer supports PEX extension: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer supports PEX extension: " + peer);
                 // peer state calls peer listener calls sendPEX()
             }
 
+            if (msgmap.get(TYPE_DHT) != null) {
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer supports DHT extension: " + peer);
+                // peer state calls peer listener calls sendDHT()
+            }
+
             MagnetState state = peer.getMagnetState();
 
             if (msgmap.get(TYPE_METADATA) == null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer does not support metadata extension: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer does not support metadata extension: " + peer);
                 // drop if we need metainfo and we haven't found anybody yet
                 synchronized(state) {
                     if (!state.isInitialized()) {
-                        _log.debug("Dropping peer, we need metadata! " + peer);
+                        log.debug("Dropping peer, we need metadata! " + peer);
                         peer.disconnect();
                     }
                 }
@@ -103,20 +119,20 @@ abstract class ExtensionHandler {
 
             BEValue msize = map.get("metadata_size");
             if (msize == null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer does not have the metainfo size yet: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer does not have the metainfo size yet: " + peer);
                 // drop if we need metainfo and we haven't found anybody yet
                 synchronized(state) {
                     if (!state.isInitialized()) {
-                        _log.debug("Dropping peer, we need metadata! " + peer);
+                        log.debug("Dropping peer, we need metadata! " + peer);
                         peer.disconnect();
                     }
                 }
                 return;
             }
             int metaSize = msize.getInt();
-            if (_log.shouldLog(Log.WARN))
-                _log.debug("Got the metainfo size: " + metaSize);
+            if (log.shouldLog(Log.DEBUG))
+                log.debug("Got the metainfo size: " + metaSize);
 
             int remaining;
             synchronized(state) {
@@ -125,21 +141,21 @@ abstract class ExtensionHandler {
 
                 if (state.isInitialized()) {
                     if (state.getSize() != metaSize) {
-                        if (_log.shouldLog(Log.WARN))
-                            _log.debug("Wrong metainfo size " + metaSize + " from: " + peer);
+                        if (log.shouldLog(Log.DEBUG))
+                            log.debug("Wrong metainfo size " + metaSize + " from: " + peer);
                         peer.disconnect();
                         return;
                     }
                 } else {
                     // initialize it
                     if (metaSize > MAX_METADATA_SIZE) {
-                        if (_log.shouldLog(Log.WARN))
-                            _log.debug("Huge metainfo size " + metaSize + " from: " + peer);
+                        if (log.shouldLog(Log.DEBUG))
+                            log.debug("Huge metainfo size " + metaSize + " from: " + peer);
                         peer.disconnect(false);
                         return;
                     }
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Initialized state, metadata size = " + metaSize + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Initialized state, metadata size = " + metaSize + " from " + peer);
                     state.initialize(metaSize);
                 }
                 remaining = state.chunksRemaining();
@@ -152,13 +168,13 @@ abstract class ExtensionHandler {
                 synchronized(state) {
                     chk = state.getNextRequest();
                 }
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Request chunk " + chk + " from " + peer);
+                if (log.shouldLog(Log.INFO))
+                    log.info("Request chunk " + chk + " from " + peer);
                 sendRequest(peer, chk);
             }
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("Handshake exception from " + peer, e);
+            if (log.shouldLog(Log.WARN))
+                log.warn("Handshake exception from " + peer, e);
         }
     }
 
@@ -166,15 +182,13 @@ abstract class ExtensionHandler {
     private static final int TYPE_DATA = 1;
     private static final int TYPE_REJECT = 2;
 
-    private static final int CHUNK_SIZE = 16*1024;
-
     /**
      * REF: BEP 9
      * @since 0.8.4
      */
-    private static void handleMetadata(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got metadata msg from " + peer);
+    private static void handleMetadata(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got metadata msg from " + peer);
         try {
             InputStream is = new ByteArrayInputStream(bs);
             BDecoder dec = new BDecoder(is);
@@ -185,8 +199,8 @@ abstract class ExtensionHandler {
 
             MagnetState state = peer.getMagnetState();
             if (type == TYPE_REQUEST) {
-                if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Got request for " + piece + " from: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Got request for " + piece + " from: " + peer);
                 byte[] pc;
                 synchronized(state) {
                     pc = state.getChunk(piece);
@@ -197,8 +211,8 @@ abstract class ExtensionHandler {
                 listener.uploaded(peer, pc.length);
             } else if (type == TYPE_DATA) {
                 int size = map.get("total_size").getInt();
-                if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Got data for " + piece + " length " + size + " from: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Got data for " + piece + " length " + size + " from: " + peer);
                 boolean done;
                 int chk = -1;
                 synchronized(state) {
@@ -207,14 +221,14 @@ abstract class ExtensionHandler {
                     int len = is.available();
                     if (len != size) {
                         // probably fatal
-                        if (_log.shouldLog(Log.WARN))
-                            _log.warn("total_size " + size + " but avail data " + len);
+                        if (log.shouldLog(Log.WARN))
+                            log.warn("total_size " + size + " but avail data " + len);
                     }
                     peer.downloaded(len);
                     listener.downloaded(peer, len);
                     done = state.saveChunk(piece, bs, bs.length - len, len);
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Got chunk " + piece + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Got chunk " + piece + " from " + peer);
                     if (!done)
                         chk = state.getNextRequest();
                 }
@@ -223,26 +237,26 @@ abstract class ExtensionHandler {
                     // Done!
                     // PeerState will call the listener (peer coord), who will
                     // check to see if the MagnetState has it
-                    if (_log.shouldLog(Log.WARN))
-                        _log.warn("Got last chunk from " + peer);
+                    if (log.shouldLog(Log.WARN))
+                        log.warn("Got last chunk from " + peer);
                 } else {
                     // get the next chunk
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Request chunk " + chk + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Request chunk " + chk + " from " + peer);
                     sendRequest(peer, chk);
                 }
             } else if (type == TYPE_REJECT) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.warn("Got reject msg from " + peer);
+                if (log.shouldLog(Log.WARN))
+                    log.warn("Got reject msg from " + peer);
                 peer.disconnect(false);
             } else {
-                if (_log.shouldLog(Log.WARN))
-                    _log.warn("Got unknown metadata msg from " + peer);
+                if (log.shouldLog(Log.WARN))
+                    log.warn("Got unknown metadata msg from " + peer);
                 peer.disconnect(false);
             }
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata ext. msg. exception from " + peer, e);
+            if (log.shouldLog(Log.INFO))
+                log.info("Metadata ext. msg. exception from " + peer, e);
             // fatal ?
             peer.disconnect(false);
         }
@@ -252,9 +266,11 @@ abstract class ExtensionHandler {
         sendMessage(peer, TYPE_REQUEST, piece);
     }
 
+  /****
     private static void sendReject(Peer peer, int piece) {
         sendMessage(peer, TYPE_REJECT, piece);
     }
+  ****/
 
     /** REQUEST and REJECT are the same except for message type */
     private static void sendMessage(Peer peer, int type, int piece) {
@@ -267,8 +283,8 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no metadata capability
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata send req msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("Metadata send req msg exception to " + peer, e);
         }
     }
 
@@ -286,8 +302,8 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no metadata caps
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata send piece msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("Metadata send piece msg exception to " + peer, e);
         }
     }
 
@@ -301,15 +317,18 @@ abstract class ExtensionHandler {
      * added.f and dropped unsupported
      * @since 0.8.4
      */
-    private static void handlePEX(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got PEX msg from " + peer);
+    private static void handlePEX(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got PEX msg from " + peer);
         try {
             InputStream is = new ByteArrayInputStream(bs);
             BDecoder dec = new BDecoder(is);
             BEValue bev = dec.bdecodeMap();
             Map<String, BEValue> map = bev.getMap();
-            byte[] ids = map.get("added").getBytes();
+            bev = map.get("added");
+            if (bev == null)
+                return;
+            byte[] ids = bev.getBytes();
             if (ids.length < HASH_LENGTH)
                 return;
             int len = Math.min(ids.length, (I2PSnarkUtil.MAX_CONNECTIONS - 1) * HASH_LENGTH);
@@ -319,14 +338,36 @@ abstract class ExtensionHandler {
                 System.arraycopy(ids, off, hash, 0, HASH_LENGTH);
                 if (DataHelper.eq(hash, peer.getPeerID().getDestHash()))
                     continue;
-                PeerID pID = new PeerID(hash);
+                PeerID pID = new PeerID(hash, listener.getUtil());
                 peers.add(pID);
             }
             // could include ourselves, listener must remove
             listener.gotPeers(peer, peers);
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.info("PEX msg exception from " + peer, e);
+            if (log.shouldLog(Log.INFO))
+                log.info("PEX msg exception from " + peer, e);
+            //peer.disconnect(false);
+        }
+    }
+
+    /**
+     * Receive the DHT port numbers
+     * @since DHT
+     */
+    private static void handleDHT(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got DHT msg from " + peer);
+        try {
+            InputStream is = new ByteArrayInputStream(bs);
+            BDecoder dec = new BDecoder(is);
+            BEValue bev = dec.bdecodeMap();
+            Map<String, BEValue> map = bev.getMap();
+            int qport = map.get("port").getInt();
+            int rport = map.get("rport").getInt();
+            listener.gotPort(peer, qport, rport);
+        } catch (Exception e) {
+            if (log.shouldLog(Log.INFO))
+                log.info("DHT msg exception from " + peer, e);
             //peer.disconnect(false);
         }
     }
@@ -353,9 +394,27 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no PEX caps
-            if (_log.shouldLog(Log.WARN))
-                _log.info("PEX msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("PEX msg exception to " + peer, e);
         }
     }
 
+    /**
+     *  Send the DHT port numbers
+     *  @since DHT
+     */
+    public static void sendDHT(Peer peer, int qport, int rport) {
+        Map<String, Object> map = new HashMap();
+        map.put("port", Integer.valueOf(qport));
+        map.put("rport", Integer.valueOf(rport));
+        byte[] payload = BEncoder.bencode(map);
+        try {
+            int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_DHT).getInt();
+            peer.sendExtension(hisMsgCode, payload);
+        } catch (Exception e) {
+            // NPE, no DHT caps
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("DHT msg exception to " + peer, e);
+        }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java

@@ -3,6 +3,7 @@ package org.klomp.snark;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -20,6 +21,7 @@ import net.i2p.client.streaming.I2PSocket;
 import net.i2p.client.streaming.I2PSocketEepGet;
 import net.i2p.client.streaming.I2PSocketManager;
 import net.i2p.client.streaming.I2PSocketManagerFactory;
+import net.i2p.client.streaming.I2PSocketOptions;
 import net.i2p.data.Base32;
 import net.i2p.data.DataFormatException;
 import net.i2p.data.Destination;
@@ -35,7 +37,7 @@ import net.i2p.util.SimpleTimer;
 import net.i2p.util.Translate;
 
 import org.klomp.snark.dht.DHT;
-//import org.klomp.snark.dht.KRPC;
+import org.klomp.snark.dht.KRPC;
 
 /**
  * I2P specific helpers for I2PSnark
@@ -44,35 +46,39 @@ import org.klomp.snark.dht.DHT;
  * (but not multiple SnarkManagers, it is still static)
  */
 public class I2PSnarkUtil {
-    private I2PAppContext _context;
-    private Log _log;
+    private final I2PAppContext _context;
+    private final Log _log;
     
     private boolean _shouldProxy;
     private String _proxyHost;
     private int _proxyPort;
     private String _i2cpHost;
     private int _i2cpPort;
-    private Map<String, String> _opts;
-    private I2PSocketManager _manager;
+    private final Map<String, String> _opts;
+    private volatile I2PSocketManager _manager;
     private boolean _configured;
+    private volatile boolean _connecting;
     private final Set<Hash> _shitlist;
     private int _maxUploaders;
     private int _maxUpBW;
     private int _maxConnections;
-    private File _tmpDir;
+    private final File _tmpDir;
     private int _startupDelay;
     private boolean _shouldUseOT;
+    private boolean _shouldUseDHT;
+    private boolean _areFilesPublic;
+    private List<String> _openTrackers;
     private DHT _dht;
 
+    private static final int EEPGET_CONNECT_TIMEOUT = 45*1000;
+    private static final int EEPGET_CONNECT_TIMEOUT_SHORT = 5*1000;
     public static final int DEFAULT_STARTUP_DELAY = 3;
-    public static final String PROP_USE_OPENTRACKERS = "i2psnark.useOpentrackers";
     public static final boolean DEFAULT_USE_OPENTRACKERS = true;
-    public static final String PROP_OPENTRACKERS = "i2psnark.opentrackers";
     public static final String DEFAULT_OPENTRACKERS = "http://tracker.welterde.i2p/a";
     public static final int DEFAULT_MAX_UP_BW = 8;  //KBps
     public static final int MAX_CONNECTIONS = 16; // per torrent
-    private static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond";
-    //private static final boolean ENABLE_DHT = true;
+    public static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond";
+    public static final boolean DEFAULT_USE_DHT = false;
 
     public I2PSnarkUtil(I2PAppContext ctx) {
         _context = ctx;
@@ -81,12 +87,14 @@ public class I2PSnarkUtil {
         //setProxy("127.0.0.1", 4444);
         setI2CPConfig("127.0.0.1", 7654, null);
         _shitlist = new ConcurrentHashSet();
-        _configured = false;
         _maxUploaders = Snark.MAX_TOTAL_UPLOADERS;
         _maxUpBW = DEFAULT_MAX_UP_BW;
         _maxConnections = MAX_CONNECTIONS;
         _startupDelay = DEFAULT_STARTUP_DELAY;
         _shouldUseOT = DEFAULT_USE_OPENTRACKERS;
+        // FIXME split if default has more than one
+        _openTrackers = Collections.singletonList(DEFAULT_OPENTRACKERS);
+        _shouldUseDHT = DEFAULT_USE_DHT;
         // This is used for both announce replies and .torrent file downloads,
         // so it must be available even if not connected to I2CP.
         // so much for multiple instances
@@ -115,6 +123,9 @@ public class I2PSnarkUtil {
     }
 ******/
     
+    /** @since 0.9.1 */
+    public I2PAppContext getContext() { return _context; }
+    
     public boolean configured() { return _configured; }
     
     public void setI2CPConfig(String i2cpHost, int i2cpPort, Map opts) {
@@ -125,6 +136,8 @@ public class I2PSnarkUtil {
         // can't remove any options this way...
         if (opts != null)
             _opts.putAll(opts);
+        // this updates the session options and tells the router
+        setMaxUpBW(_maxUpBW);
         _configured = true;
     }
     
@@ -134,6 +147,7 @@ public class I2PSnarkUtil {
     }
     
     /**
+     *  This updates the session options and tells the router
      *  @param limit KBps
      */
     public void setMaxUpBW(int limit) {
@@ -175,11 +189,21 @@ public class I2PSnarkUtil {
     public int getMaxConnections() { return _maxConnections; }
     public int getStartupDelay() { return _startupDelay; }  
   
+    /** @since 0.8.9 */
+    public boolean getFilesPublic() { return _areFilesPublic; }
+  
+    /** @since 0.8.9 */
+    public void setFilesPublic(boolean yes) { _areFilesPublic = yes; }
+
+    /** @since 0.9.1 */
+    public File getTempDir() { return _tmpDir; }
+
     /**
      * Connect to the router, if we aren't already
      */
     synchronized public boolean connect() {
         if (_manager == null) {
+            _connecting = true;
             // try to find why reconnecting after stop
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Connecting to I2P", new Exception("I did it"));
@@ -198,6 +222,8 @@ public class I2PSnarkUtil {
             // we don't need fast handshake for peer connections.
             //if (opts.getProperty("i2p.streaming.connectDelay") == null)
             //    opts.setProperty("i2p.streaming.connectDelay", "500");
+            if (opts.getProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT) == null)
+                opts.setProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT, "75000");
             if (opts.getProperty("i2p.streaming.inactivityTimeout") == null)
                 opts.setProperty("i2p.streaming.inactivityTimeout", "240000");
             if (opts.getProperty("i2p.streaming.inactivityAction") == null)
@@ -210,11 +236,19 @@ public class I2PSnarkUtil {
             //    opts.setProperty("i2p.streaming.writeTimeout", "90000");
             //if (opts.getProperty("i2p.streaming.readTimeout") == null)
             //    opts.setProperty("i2p.streaming.readTimeout", "120000");
+            if (opts.getProperty("i2p.streaming.maxConnsPerMinute") == null)
+                opts.setProperty("i2p.streaming.maxConnsPerMinute", "2");
+            if (opts.getProperty("i2p.streaming.maxTotalConnsPerMinute") == null)
+                opts.setProperty("i2p.streaming.maxTotalConnsPerMinute", "8");
+            if (opts.getProperty("i2p.streaming.maxConnsPerHour") == null)
+                opts.setProperty("i2p.streaming.maxConnsPerHour", "20");
+            if (opts.getProperty("i2p.streaming.enforceProtocol") == null)
+                opts.setProperty("i2p.streaming.enforceProtocol", "true");
             _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts);
+            _connecting = false;
         }
-        // FIXME this only instantiates krpc once, left stuck with old manager
-        //if (ENABLE_DHT && _manager != null && _dht == null)
-        //    _dht = new KRPC(_context, _manager.getSession());
+        if (_shouldUseDHT && _manager != null && _dht == null)
+            _dht = new KRPC(_context, _manager.getSession());
         return (_manager != null);
     }
     
@@ -226,15 +260,35 @@ public class I2PSnarkUtil {
 
     public boolean connected() { return _manager != null; }
 
+    /** @since 0.9.1 */
+    public boolean isConnecting() { return _manager == null && _connecting; }
+
+    /**
+     *  For FetchAndAdd
+     *  @return null if not connected
+     *  @since 0.9.1
+     */
+    public I2PSocketManager getSocketManager() {
+        return _manager;
+    }
+
     /**
      * Destroy the destination itself
      */
-    public void disconnect() {
+    public synchronized void disconnect() {
+        if (_dht != null) {
+            _dht.stop();
+            _dht = null;
+        }
         I2PSocketManager mgr = _manager;
         // FIXME this can cause race NPEs elsewhere
         _manager = null;
         _shitlist.clear();
-        mgr.destroySocketManager();
+        if (mgr != null) {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Disconnecting from I2P", new Exception("I did it"));
+            mgr.destroySocketManager();
+        }
         // this will delete a .torrent file d/l in progress so don't do that...
         FileUtil.rmdir(_tmpDir, false);
         // in case the user will d/l a .torrent file next...
@@ -261,7 +315,7 @@ public class I2PSnarkUtil {
             return rv;
         } catch (I2PException ie) {
             _shitlist.add(dest);
-            SimpleScheduler.getInstance().addEvent(new Unshitlist(dest), 10*60*1000);
+            _context.simpleScheduler().addEvent(new Unshitlist(dest), 10*60*1000);
             throw new IOException("Unable to reach the peer " + peer + ": " + ie.getMessage());
         }
     }
@@ -273,11 +327,24 @@ public class I2PSnarkUtil {
     }
     
     /**
-     * fetch the given URL, returning the file it is stored in, or null on error
+     * Fetch the given URL, returning the file it is stored in, or null on error.
+     * No retries.
      */
     public File get(String url) { return get(url, true, 0); }
+
+    /**
+     * @param rewrite if true, convert http://KEY.i2p/foo/announce to http://i2p/KEY/foo/announce
+     */
     public File get(String url, boolean rewrite) { return get(url, rewrite, 0); }
+
+    /**
+     * @param retries if < 0, set timeout to a few seconds
+     */
     public File get(String url, int retries) { return get(url, true, retries); }
+
+    /**
+     * @param retries if < 0, set timeout to a few seconds
+     */
     public File get(String url, boolean rewrite, int retries) {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Fetching [" + url + "] proxy=" + _proxyHost + ":" + _proxyPort + ": " + _shouldProxy);
@@ -286,7 +353,7 @@ public class I2PSnarkUtil {
             // we could use the system tmp dir but deleteOnExit() doesn't seem to work on all platforms...
             out = SecureFile.createTempFile("i2psnark", null, _tmpDir);
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("temp file error", ioe);
             if (out != null)
                 out.delete();
             return null;
@@ -298,12 +365,21 @@ public class I2PSnarkUtil {
         //_log.debug("Rewritten url [" + fetchURL + "]");
         //EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, retries, out.getAbsolutePath(), fetchURL);
         // Use our tunnel for announces and .torrent fetches too! Make sure we're connected first...
-        if (!connected()) {
-            if (!connect())
+        int timeout;
+        if (retries < 0) {
+            if (!connected())
                 return null;
+            timeout = EEPGET_CONNECT_TIMEOUT_SHORT;
+            retries = 0;
+        } else {
+            timeout = EEPGET_CONNECT_TIMEOUT;
+            if (!connected()) {
+                if (!connect())
+                    return null;
+            }
         }
         EepGet get = new I2PSocketEepGet(_context, _manager, retries, out.getAbsolutePath(), fetchURL);
-        if (get.fetch()) {
+        if (get.fetch(timeout)) {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Fetch successful [" + url + "]: size=" + out.length());
             return out;
@@ -377,7 +453,8 @@ public class I2PSnarkUtil {
                     if (sess != null) {
                         byte[] b = Base32.decode(ip.substring(0, BASE32_HASH_LENGTH));
                         if (b != null) {
-                            Hash h = new Hash(b);
+                            //Hash h = new Hash(b);
+                            Hash h = Hash.create(b);
                             if (_log.shouldLog(Log.INFO))
                                 _log.info("Using existing session for lookup of " + ip);
                             try {
@@ -432,31 +509,17 @@ public class I2PSnarkUtil {
     }
     
     /** @param ot non-null */
-    public void setOpenTrackerString(String ot) { 
-        _opts.put(PROP_OPENTRACKERS, ot);
+    public void setOpenTrackers(List<String> ot) { 
+        _openTrackers = ot;
     }
 
-    public String getOpenTrackerString() { 
-        String rv = (String) _opts.get(PROP_OPENTRACKERS);
-        if (rv == null)
-            return DEFAULT_OPENTRACKERS;
-        return rv;
-    }
-
-    /** comma delimited list open trackers to use as backups */
-    /** sorted map of name to announceURL=baseURL */
+    /** List of open trackers to use as backups
+     *  @return non-null, possibly unmodifiable, empty if disabled
+     */
     public List<String> getOpenTrackers() { 
         if (!shouldUseOpenTrackers())
-            return null;
-        List<String> rv = new ArrayList(1);
-        String trackers = getOpenTrackerString();
-        StringTokenizer tok = new StringTokenizer(trackers, ", ");
-        while (tok.hasMoreTokens())
-            rv.add(tok.nextToken());
-        
-        if (rv.isEmpty())
-            return null;
-        return rv;
+            return Collections.EMPTY_LIST;
+        return _openTrackers;
     }
     
     public void setUseOpenTrackers(boolean yes) {
@@ -466,6 +529,22 @@ public class I2PSnarkUtil {
     public boolean shouldUseOpenTrackers() {
         return _shouldUseOT;
     }
+    
+    /** @since DHT */
+    public synchronized void setUseDHT(boolean yes) {
+        _shouldUseDHT = yes;
+        if (yes && _manager != null && _dht == null) {
+            _dht = new KRPC(_context, _manager.getSession());
+        } else if (!yes && _dht != null) {
+            _dht.stop();
+            _dht = null;
+        }
+    }
+
+    /** @since DHT */
+    public boolean shouldUseDHT() {
+        return _shouldUseDHT;
+    }
 
     /**
      *  Like DataHelper.toHexString but ensures no loss of leading zero bytes
@@ -482,40 +561,6 @@ public class I2PSnarkUtil {
         return buf.toString();
     }
 
-    /** hook between snark's logger and an i2p log */
-    void debug(String msg, int snarkDebugLevel) {
-        debug(msg, snarkDebugLevel, null);
-    }
-    void debug(String msg, int snarkDebugLevel, Throwable t) {
-        if (t instanceof OutOfMemoryError) {
-            try { Thread.sleep(100); } catch (InterruptedException ie) {}
-            try {
-                t.printStackTrace();
-            } catch (Throwable tt) {}
-            try {
-                System.out.println("OOM thread: " + Thread.currentThread().getName());
-            } catch (Throwable tt) {}
-        }
-        switch (snarkDebugLevel) {
-            case 0:
-            case 1:
-                _log.error(msg, t);
-                break;
-            case 2:
-                _log.warn(msg, t);
-                break;
-            case 3:
-            case 4:
-                _log.info(msg, t);
-                break;
-            case 5:
-            case 6:
-            default:
-                _log.debug(msg, t);
-                break;
-        }
-    }
-
     private static final String BUNDLE_NAME = "org.klomp.snark.web.messages";
 
     /** lang in routerconsole.lang property, else current locale */

apps/i2psnark/java/src/org/klomp/snark/MagnetState.java

@@ -5,10 +5,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Random;
 
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
+import net.i2p.util.RandomSource;
 
 import org.klomp.snark.bencode.BDecoder;
 import org.klomp.snark.bencode.BEValue;
@@ -27,7 +27,6 @@ import org.klomp.snark.bencode.BEValue;
  */
 class MagnetState {
     public static final int CHUNK_SIZE = 16*1024;
-    private static final Random random = I2PAppContext.getGlobalContext().random();
 
     private final byte[] infohash;
     private boolean complete;
@@ -129,7 +128,7 @@ class MagnetState {
             throw new IllegalArgumentException("not initialized");
         if (complete)
             throw new IllegalArgumentException("complete");
-        int rand = random.nextInt(totalChunks);
+        int rand = RandomSource.getInstance().nextInt(totalChunks);
         for (int i = 0; i < totalChunks; i++) {
             int chk = (i + rand) % totalChunks; 
             if (!(have.get(chk) || requested.get(chk))) {

apps/i2psnark/java/src/org/klomp/snark/Message.java

@@ -23,8 +23,6 @@ package org.klomp.snark;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-import net.i2p.util.SimpleTimer;
-
 // Used to queue outgoing connections
 // sendMessage() should be used to translate them to wire format.
 class Message

apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java

@@ -33,7 +33,6 @@ import java.util.Map;
 
 import net.i2p.I2PAppContext;
 import net.i2p.crypto.SHA1;
-import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
 import net.i2p.util.Log;
 
@@ -61,6 +60,7 @@ public class MetaInfo
   private final int piece_length;
   private final byte[] piece_hashes;
   private final long length;
+  private final boolean privateTorrent;
   private Map<String, BEValue> infoMap;
 
   /**
@@ -71,7 +71,7 @@ public class MetaInfo
    *  @param lengths null for single-file torrent
    */
   MetaInfo(String announce, String name, String name_utf8, List<List<String>> files, List<Long> lengths,
-           int piece_length, byte[] piece_hashes, long length)
+           int piece_length, byte[] piece_hashes, long length, boolean privateTorrent)
   {
     this.announce = announce;
     this.name = name;
@@ -82,6 +82,7 @@ public class MetaInfo
     this.piece_length = piece_length;
     this.piece_hashes = piece_hashes;
     this.length = length;
+    this.privateTorrent = privateTorrent;
 
     // TODO if we add a parameter for other keys
     //if (other != null) {
@@ -160,6 +161,10 @@ public class MetaInfo
     else
         name_utf8 = null;
 
+    // BEP 27
+    val = info.get("private");
+    privateTorrent = val != null && val.getString().equals("1");
+
     val = info.get("piece length");
     if (val == null)
         throw new InvalidBEncodingException("Missing piece length number");
@@ -212,7 +217,7 @@ public class MetaInfo
             if (l < oldTotal)
               throw new InvalidBEncodingException("Huge total length");
 
-            val = (BEValue)desc.get("path");
+            val = desc.get("path");
             if (val == null)
               throw new InvalidBEncodingException("Missing path list");
             List<BEValue> path_list = val.getList();
@@ -238,7 +243,7 @@ public class MetaInfo
 
             m_files.add(Collections.unmodifiableList(file));
             
-            val = (BEValue)desc.get("path.utf-8");
+            val = desc.get("path.utf-8");
             if (val != null) {
                 path_list = val.getList();
                 path_length = path_list.size();
@@ -318,6 +323,14 @@ public class MetaInfo
     return name;
   }
 
+  /**
+   * Is it a private torrent?
+   * @since 0.9
+   */
+  public boolean isPrivate() {
+    return privateTorrent;
+  }
+
   /**
    * Returns a list of lists of file name hierarchies or null if it is
    * a single name. It has the same size as the list returned by
@@ -409,6 +422,29 @@ public class MetaInfo
         return false;
     return true;
   }
+  
+  /**
+   *  @return good
+   *  @since 0.9.1
+   */
+  boolean checkPiece(PartialPiece pp) {
+    MessageDigest sha1 = SHA1.getInstance();
+    int piece = pp.getPiece();
+    byte[] hash;
+    try {
+        hash = pp.getHash();
+    } catch (IOException ioe) {
+        // Could be caused by closing a peer connnection
+        // we don't want the exception to propagate through
+        // to Storage.putPiece()
+        _log.warn("Error checking", ioe);
+        return false;
+    }
+    for (int i = 0; i < 20; i++)
+      if (hash[i] != piece_hashes[20 * piece + i])
+        return false;
+    return true;
+  }
 
   /**
    * Returns the total length of the torrent in bytes.
@@ -439,7 +475,7 @@ public class MetaInfo
   {
     return new MetaInfo(announce, name, name_utf8, files,
                         lengths, piece_length,
-                        piece_hashes, length);
+                        piece_hashes, length, privateTorrent);
   }
 
   /**
@@ -475,6 +511,10 @@ public class MetaInfo
     info.put("name", name);
     if (name_utf8 != null)
         info.put("name.utf-8", name_utf8);
+    // BEP 27
+    if (privateTorrent)
+        info.put("private", "1");
+
     info.put("piece length", Integer.valueOf(piece_length));
     info.put("pieces", piece_hashes);
     if (files == null)

apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java

@@ -1,6 +1,22 @@
 package org.klomp.snark;
 
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.security.MessageDigest;
+
+import net.i2p.I2PAppContext;
+import net.i2p.crypto.SHA1;
+import net.i2p.util.Log;
+import net.i2p.util.SecureFile;
+
 /**
+ * Store the received data either on the heap or in a temp file.
+ * The third option, to write chunks directly to the destination file,
+ * is unimplemented.
+ *
  * This is the class passed from PeerCoordinator to PeerState so
  * PeerState may start requests.
  *
@@ -8,45 +24,81 @@ package org.klomp.snark;
  * a piece is not completely downloaded, for example
  * when the Peer disconnects or chokes.
  *
+ * New objects for the same piece are created during the end game -
+ * this object should not be shared among multiple peers.
+ *
  * @since 0.8.2
  */
 class PartialPiece implements Comparable {
 
-    private final int piece;
+    // we store the piece so we can use it in compareTo()
+    private final Piece piece;
+    // null if using temp file
     private final byte[] bs;
-    private final int off;
-    private final long createdTime;
+    private int off;
+    //private final long createdTime;
+    private File tempfile;
+    private RandomAccessFile raf;
+    private final int pclen;
+    private final File tempDir;
+
+    // Any bigger than this, use temp file instead of heap
+    private static final int MAX_IN_MEM = 128 * 1024;
+    // May be reduced on OOM
+    private static int _max_in_mem = MAX_IN_MEM;
 
     /**
      * Used by PeerCoordinator.
      * Creates a new PartialPiece, with no chunks yet downloaded.
-     * Allocates the data.
+     * Allocates the data storage area, either on the heap or in the
+     * temp directory, depending on size.
      *
      * @param piece Piece number requested.
      * @param len must be equal to the piece length
      */
-    public PartialPiece (int piece, int len) throws OutOfMemoryError {
+    public PartialPiece (Piece piece, int len, File tempDir) {
         this.piece = piece;
-        this.bs = new byte[len];
-        this.off = 0;
-        this.createdTime = 0;
+        this.pclen = len;
+        //this.createdTime = 0;
+        this.tempDir = tempDir;
+
+        // temps for finals
+        byte[] tbs = null;
+        try {
+            if (len <= MAX_IN_MEM) {
+                try {
+                    tbs = new byte[len];
+                    return;
+                } catch (OutOfMemoryError oom) {
+                    if (_max_in_mem > PeerState.PARTSIZE)
+                        _max_in_mem /= 2;
+                    Log log = I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class);
+                    log.logAlways(Log.WARN, "OOM creating new partial piece");
+                    // fall through to use temp file
+                }
+            }
+            // delay creating temp file until required in read()
+        } finally {
+            // finals
+            this.bs = tbs;
+        }
     }
 
     /**
-     * Used by PeerState.
-     * Creates a new PartialPiece, with chunks up to but not including
-     * firstOutstandingRequest already downloaded and stored in the Request byte array.
+     *  Caller must synchronize
      *
-     * Note that this cannot handle gaps; chunks after a missing chunk cannot be saved.
-     * That would be harder.
-     *
-     * @param firstOutstandingRequest the first request not fulfilled for the piece
+     *  @since 0.9.1
      */
-    public PartialPiece (Request firstOutstandingRequest) {
-        this.piece = firstOutstandingRequest.piece;
-        this.bs = firstOutstandingRequest.bs;
-        this.off = firstOutstandingRequest.off;
-        this.createdTime = System.currentTimeMillis();
+    private void createTemp() throws IOException {
+        //tfile = SecureFile.createTempFile("piece", null, tempDir);
+        // debug
+        tempfile = SecureFile.createTempFile("piece_" + piece.getId() + '_', null, tempDir);
+        //I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Created " + tempfile);
+        // tfile.deleteOnExit() ???
+        raf = new RandomAccessFile(tempfile, "rw");
+        // Do not preallocate the file space.
+        // Not necessary to call setLength(), file is extended when written
+        //traf.setLength(len);
     }
 
     /**
@@ -55,33 +107,168 @@ class PartialPiece implements Comparable {
      */
 
     public Request getRequest() {
-        return new Request(this.piece, this.bs, this.off, Math.min(this.bs.length - this.off, PeerState.PARTSIZE));
+        return new Request(this, this.off, Math.min(this.pclen - this.off, PeerState.PARTSIZE));
     }
 
     /** piece number */
     public int getPiece() {
-         return this.piece;
+         return this.piece.getId();
     }
 
-    /** how many bytes are good */
+    /**
+     *  @since 0.9.1
+     */
+    public int getLength() {
+         return this.pclen;
+    }
+
+    /**
+     *  How many bytes are good - only valid by setDownloaded()
+     */
     public int getDownloaded() {
          return this.off;
     }
 
+    /**
+     *  Call this before returning a PartialPiece to the PeerCoordinator
+     *  @since 0.9.1
+     */
+    public void setDownloaded(int offset) {
+         this.off = offset;
+    }
+
+/****
     public long getCreated() {
          return this.createdTime;
     }
+****/
 
     /**
-     *  Highest downloaded first
+     *  Piece must be complete.
+     *  The SHA1 hash of the completely read data.
+     *  @since 0.9.1
+     */
+    public byte[] getHash() throws IOException {
+        MessageDigest sha1 = SHA1.getInstance();
+        if (bs != null) {
+            sha1.update(bs);
+        } else {
+            int read = 0;
+            byte[] buf = new byte[Math.min(pclen, 16384)];
+            synchronized (this) {
+                if (raf == null)
+                    throw new IOException();
+                raf.seek(0);
+                while (read < pclen) {
+                    int rd = raf.read(buf, 0, Math.min(buf.length, pclen - read));
+                    if (rd < 0)
+                        break;
+                    read += rd;
+                    sha1.update(buf, 0, rd);
+                }
+            }
+            if (read < pclen)
+                throw new IOException();
+        }
+        return sha1.digest();
+    }
+    
+    /**
+     *  Blocking.
+     *  @since 0.9.1
+     */
+    public void read(DataInputStream din, int off, int len) throws IOException {
+        if (bs != null) {
+            din.readFully(bs, off, len);
+        } else {
+            // read in fully before synching on raf
+            byte[] tmp = new byte[len];
+            din.readFully(tmp);
+            synchronized (this) {
+                if (raf == null)
+                    createTemp();
+                raf.seek(off);
+                raf.write(tmp);
+            }
+        }
+    }
+
+    /**
+     *  Piece must be complete.
+     *  Caller must synchronize on out and seek to starting point.
+     *  Caller must call release() when done with the whole piece.
+     *
+     *  @param out stream to write to
+     *  @param offset offset in the piece
+     *  @param len length to write
+     *  @since 0.9.1
+     */
+    public void write(DataOutput out, int offset, int len) throws IOException {
+        if (bs != null) {
+            out.write(bs, offset, len);
+        } else {
+            int read = 0;
+            byte[] buf = new byte[Math.min(len, 16384)];
+            synchronized (this) {
+                if (raf == null)
+                    throw new IOException();
+                raf.seek(offset);
+                while (read < len) {
+                    int rd = Math.min(buf.length, len - read);
+                    raf.readFully(buf, 0, rd);
+                    read += rd;
+                    out.write(buf, 0, rd);
+                }
+            }
+        }
+    }
+    
+    /**
+     *  Release all resources.
+     *
+     *  @since 0.9.1
+     */
+    public void release() {
+        if (bs == null) {
+            synchronized (this) {
+                if (raf != null)
+                    locked_release();
+            }
+            //if (raf != null)
+            //    I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Released " + tempfile);
+        }
+    }
+    
+    /**
+     *  Caller must synchronize
+     *
+     *  @since 0.9.1
+     */
+    private void locked_release() {
+        try {
+            raf.close();
+        } catch (IOException ioe) {
+            I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Error closing " + raf, ioe);
+        }
+        tempfile.delete();
+    }
+
+    /*
+     *  Highest priority first,
+     *  then rarest first,
+     *  then highest downloaded first
      */
     public int compareTo(Object o) throws ClassCastException {
-        return ((PartialPiece)o).off - this.off;  // reverse
+        PartialPiece opp = (PartialPiece)o;
+        int d = this.piece.compareTo(opp.piece);
+        if (d != 0)
+            return d;
+        return opp.off - this.off;  // reverse
     }
     
     @Override
     public int hashCode() {
-        return piece * 7777;
+        return piece.getId() * 7777;
     }
 
     /**
@@ -92,13 +279,13 @@ class PartialPiece implements Comparable {
     public boolean equals(Object o) {
         if (o instanceof PartialPiece) {
             PartialPiece pp = (PartialPiece)o;
-            return pp.piece == this.piece;
+            return pp.piece.getId() == this.piece.getId();
         }
         return false;
     }
 
     @Override
     public String toString() {
-        return "Partial(" + piece + ',' + off + ',' + bs.length + ')';
+        return "Partial(" + piece.getId() + ',' + off + ',' + pclen + ')';
     }
 }

apps/i2psnark/java/src/org/klomp/snark/Peer.java

@@ -80,7 +80,9 @@ public class Peer implements Comparable
   static final long OPTION_FAST      = 0x0000000000000004l;
   static final long OPTION_DHT       = 0x0000000000000001l;
   /** we use a different bit since the compact format is different */
+/* no, let's use an extension message
   static final long OPTION_I2P_DHT   = 0x0000000040000000l;
+*/
   static final long OPTION_AZMP      = 0x1000000000000000l;
   private long options;
 
@@ -268,15 +270,18 @@ public class Peer implements Comparable
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Peer supports extensions, sending reply message");
             int metasize = metainfo != null ? metainfo.getInfoBytes().length : -1;
-            out.sendExtension(0, ExtensionHandler.getHandshake(metasize));
+            boolean pexAndMetadata = metainfo == null || !metainfo.isPrivate();
+            boolean dht = util.getDHT() != null;
+            out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata, dht));
         }
 
-        if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) {
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Peer supports DHT, sending PORT message");
-            int port = util.getDHT().getPort();
-            out.sendPort(port);
-        }
+        // Old DHT PORT message
+        //if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) {
+        //    if (_log.shouldLog(Log.DEBUG))
+        //        _log.debug("Peer supports DHT, sending PORT message");
+        //    int port = util.getDHT().getPort();
+        //    out.sendPort(port);
+        //}
 
         // Send our bitmap
         if (bitfield != null)
@@ -459,7 +464,7 @@ public class Peer implements Comparable
         if (this.deregister) {
           PeerListener p = s.listener;
           if (p != null) {
-            List<PartialPiece> pcs = s.returnPartialPieces();
+            List<Request> pcs = s.returnPartialPieces();
             if (!pcs.isEmpty())
                 p.savePartialPieces(this, pcs);
             // now covered by savePartialPieces

apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java

@@ -46,6 +46,10 @@ public class PeerAcceptor
   private final PeerCoordinator coordinator;
   final PeerCoordinatorSet coordinators;
 
+  /** shorten timeout while reading handshake */
+  private static final long HASH_READ_TIMEOUT = 45*1000;
+
+
   public PeerAcceptor(PeerCoordinator coordinator)
   {
     this.coordinator = coordinator;
@@ -69,11 +73,20 @@ public class PeerAcceptor
     // talk about, and we can just look for that in our list of active torrents.
     byte peerInfoHash[] = null;
     if (in instanceof BufferedInputStream) {
+        // multitorrent
         in.mark(LOOKAHEAD_SIZE);
-        peerInfoHash = readHash(in);
+        long timeout = socket.getReadTimeout();
+        socket.setReadTimeout(HASH_READ_TIMEOUT);
+        try {
+            peerInfoHash = readHash(in);
+        } catch (IOException ioe) {
+            // unique exception so ConnectionAcceptor can blame the peer
+            throw new ProtocolException(ioe.toString());
+        }
+        socket.setReadTimeout(timeout);
         in.reset();
     } else {
-        // is this working right?
+        // Single torrent - is this working right?
         try {
           peerInfoHash = readHash(in);
           if (_log.shouldLog(Log.INFO))
@@ -104,9 +117,8 @@ public class PeerAcceptor
         }
     } else {
         // multitorrent capable, so lets see what we can handle
-        for (Iterator iter = coordinators.iterator(); iter.hasNext(); ) {
-            PeerCoordinator cur = (PeerCoordinator)iter.next();
-
+        PeerCoordinator cur = coordinators.get(peerInfoHash);
+        if (cur != null) {
             if (DataHelper.eq(cur.getInfoHash(), peerInfoHash)) {
                 if (cur.needPeers())
                   {
@@ -130,22 +142,50 @@ public class PeerAcceptor
     }
   }
 
+  private static final String PROTO_STR = "BitTorrent protocol";
+  private static final int PROTO_STR_LEN = PROTO_STR.length();
+  private static final int PROTO_LEN = PROTO_STR_LEN + 1;
+  private static final int[] PROTO = new int[PROTO_LEN];
+  static {
+      PROTO[0] = PROTO_STR_LEN;
+      for (int i = 0; i < PROTO_STR_LEN; i++) {
+          PROTO[i+1] = PROTO_STR.charAt(i);
+      }
+  }
+
   /** 48 */
-  private static final int LOOKAHEAD_SIZE = 1 + // chr(19)
-                                            "BitTorrent protocol".length() +
+  private static final int LOOKAHEAD_SIZE = PROTO_LEN +
                                             8 + // blank, reserved
                                             20; // infohash
 
   /** 
-   * Read ahead to the infohash, throwing an exception if there isn't enough data
+   * Read ahead to the infohash, throwing an exception if there isn't enough data.
+   * Also check the first 20 bytes for the correct protocol here and throw IOE if bad,
+   * so we don't hang waiting for 48 bytes if it's not a bittorrent client.
+   * The 20 bytes are checked again in Peer.handshake().
    */
-  private byte[] readHash(InputStream in) throws IOException {
-    byte buf[] = new byte[LOOKAHEAD_SIZE];
+  private static byte[] readHash(InputStream in) throws IOException {
+    for (int i = 0; i < PROTO_LEN; i++) {
+        int b = in.read();
+        if (b != PROTO[i])
+            throw new IOException("Bad protocol 0x" + Integer.toHexString(b) + " at byte " + i);
+    }
+    if (in.skip(8) != 8)
+        throw new IOException("EOF before hash");
+    byte buf[] = new byte[20];
     int read = DataHelper.read(in, buf);
     if (read != buf.length)
         throw new IOException("Unable to read the hash (read " + read + ")");
-    byte rv[] = new byte[20];
-    System.arraycopy(buf, buf.length-rv.length, rv, 0, rv.length);
-    return rv;
+    return buf;
   }
+
+    /** 
+     *  A unique exception so we can tell the ConnectionAcceptor about non-BT connections
+     *  @since 0.9.1
+     */
+    public static class ProtocolException extends IOException {
+        public ProtocolException(String s) {
+            super(s);
+        }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java

@@ -21,11 +21,14 @@
 package org.klomp.snark;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.util.Log;
+
+import org.klomp.snark.dht.DHT;
 
 /**
  * TimerTask that checks for good/bad up/downloader. Works together
@@ -37,16 +40,18 @@ class PeerCheckerTask implements Runnable
 
   private final PeerCoordinator coordinator;
   private final I2PSnarkUtil _util;
+  private final Log _log;
+  private final Random random;
   private int _runCount;
 
   PeerCheckerTask(I2PSnarkUtil util, PeerCoordinator coordinator)
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(PeerCheckerTask.class);
+    random = util.getContext().random();
     this.coordinator = coordinator;
   }
 
-  private static final Random random = I2PAppContext.getGlobalContext().random();
-
   public void run()
   {
         _runCount++;
@@ -60,9 +65,7 @@ class PeerCheckerTask implements Runnable
         long worstdownload = Long.MAX_VALUE;
         Peer worstDownloader = null;
 
-        int peers = 0;
         int uploaders = 0;
-        int downloaders = 0;
         int removedCount = 0;
 
         long uploaded = 0;
@@ -73,6 +76,7 @@ class PeerCheckerTask implements Runnable
         List<Peer> removed = new ArrayList();
         int uploadLimit = coordinator.allowedUploaders();
         boolean overBWLimit = coordinator.overUpBWLimit();
+        DHT dht = _util.getDHT();
         for (Peer peer : peerList) {
 
             // Remove dying peers
@@ -85,12 +89,16 @@ class PeerCheckerTask implements Runnable
                 continue;
               }
 
-            peers++;
+            if (peer.getInactiveTime() > PeerCoordinator.MAX_INACTIVE) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Disconnecting peer idle " +
+                              DataHelper.formatDuration(peer.getInactiveTime()) + ": " + peer);
+                peer.disconnect();
+                continue;
+            }
 
             if (!peer.isChoking())
               uploaders++;
-            if (!peer.isChoked() && peer.isInteresting())
-              downloaders++;
 
             long upload = peer.getUploaded();
             uploaded += upload;
@@ -99,14 +107,15 @@ class PeerCheckerTask implements Runnable
 	    peer.setRateHistory(upload, download);
             peer.resetCounters();
 
-            _util.debug(peer + ":", Snark.DEBUG);
-            _util.debug(" ul: " + upload*1024/KILOPERSECOND
+            if (_log.shouldLog(Log.DEBUG)) {
+                _log.debug(peer + ":"
+                        + " ul: " + upload*1024/KILOPERSECOND
                         + " dl: " + download*1024/KILOPERSECOND
                         + " i: " + peer.isInterested()
                         + " I: " + peer.isInteresting()
                         + " c: " + peer.isChoking()
-                        + " C: " + peer.isChoked(),
-                        Snark.DEBUG);
+                        + " C: " + peer.isChoked());
+            }
 
             // Choke a percentage of them rather than all so it isn't so drastic...
             // unless this torrent is over the limit all by itself.
@@ -127,8 +136,8 @@ class PeerCheckerTask implements Runnable
                 // Check if it still wants pieces from us.
                 if (!peer.isInterested())
                   {
-                    _util.debug("Choke uninterested peer: " + peer,
-                                Snark.INFO);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke uninterested peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -138,8 +147,8 @@ class PeerCheckerTask implements Runnable
                   }
                 else if (overBWLimitChoke)
                   {
-                    _util.debug("BW limit (" + upload + "/" + uploaded + "), choke peer: " + peer,
-                                Snark.INFO);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("BW limit (" + upload + "/" + uploaded + "), choke peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -151,7 +160,8 @@ class PeerCheckerTask implements Runnable
                 else if (peer.isInteresting() && peer.isChoked())
                   {
                     // If they are choking us make someone else a downloader
-                    _util.debug("Choke choking peer: " + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke choking peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -163,7 +173,8 @@ class PeerCheckerTask implements Runnable
                 else if (!peer.isInteresting() && !coordinator.completed())
                   {
                     // If they aren't interesting make someone else a downloader
-                    _util.debug("Choke uninteresting peer: " + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke uninteresting peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -177,8 +188,8 @@ class PeerCheckerTask implements Runnable
                          && download == 0)
                   {
                     // We are downloading but didn't receive anything...
-                    _util.debug("Choke downloader that doesn't deliver:"
-                                + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke downloader that doesn't deliver: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -205,10 +216,13 @@ class PeerCheckerTask implements Runnable
             // send PEX
             if ((_runCount % 17) == 0 && !peer.isCompleted())
                 coordinator.sendPeers(peer);
-            peer.keepAlive();
+            // cheap failsafe for seeds connected to seeds, stop pinging and hopefully
+            // the inactive checker (above) will eventually disconnect it
+            if (coordinator.getNeededLength() > 0 || !peer.isCompleted())
+                peer.keepAlive();
             // announce them to local tracker (TrackerClient does this too)
-            if (_util.getDHT() != null && (_runCount % 5) == 0) {
-                _util.getDHT().announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash());
+            if (dht != null && (_runCount % 5) == 0) {
+                dht.announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash());
             }
           }
 
@@ -222,8 +236,8 @@ class PeerCheckerTask implements Runnable
             || uploaders > uploadLimit)
             && worstDownloader != null)
           {
-            _util.debug("Choke worst downloader: " + worstDownloader,
-                        Snark.DEBUG);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Choke worst downloader: " + worstDownloader);
 
             worstDownloader.setChoking(true);
             coordinator.uploaders--;
@@ -256,8 +270,8 @@ class PeerCheckerTask implements Runnable
         }
 
         // announce ourselves to local tracker (TrackerClient does this too)
-        if (_util.getDHT() != null && (_runCount % 16) == 0) {
-            _util.getDHT().announce(coordinator.getInfoHash());
+        if (dht != null && (_runCount % 16) == 0) {
+            dht.announce(coordinator.getInfoHash());
         }
   }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java

@@ -148,11 +148,9 @@ class PeerConnectionIn implements Runnable
                 begin = din.readInt();
                 len = i-9;
                 Request req = ps.getOutstandingRequest(piece, begin, len);
-                byte[] piece_bytes;
                 if (req != null)
                   {
-                    piece_bytes = req.bs;
-                    din.readFully(piece_bytes, begin, len);
+                    req.read(din);
                     ps.pieceMessage(req);
                     if (_log.shouldLog(Log.DEBUG)) 
                         _log.debug("Received data(" + piece + "," + begin + ") from " + peer);
@@ -160,8 +158,9 @@ class PeerConnectionIn implements Runnable
                 else
                   {
                     // XXX - Consume but throw away afterwards.
-                    piece_bytes = new byte[len];
-                    din.readFully(piece_bytes);
+                    int rcvd = din.skipBytes(len);
+                    if (rcvd != len)
+                        throw new IOException("EOF reading unwanted data");
                     if (_log.shouldLog(Log.DEBUG)) 
                         _log.debug("Received UNWANTED data(" + piece + "," + begin + ") from " + peer);
                   }

apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java

@@ -143,7 +143,7 @@ class PeerConnectionOut implements Runnable
                           }
                       }
                     if (m == null && !sendQueue.isEmpty()) {
-                      m = (Message)sendQueue.remove(0);
+                      m = sendQueue.remove(0);
                       //SimpleTimer.getInstance().removeEvent(m.expireEvent);
                     }
                   }
@@ -395,7 +395,7 @@ class PeerConnectionOut implements Runnable
         while (it.hasNext())
           {
             Message m = (Message)it.next();
-            if (m.type == Message.REQUEST && m.piece == req.piece &&
+            if (m.type == Message.REQUEST && m.piece == req.getPiece() &&
                 m.begin == req.off && m.length == req.len)
               {
                 if (_log.shouldLog(Log.DEBUG))
@@ -406,7 +406,7 @@ class PeerConnectionOut implements Runnable
       }
     Message m = new Message();
     m.type = Message.REQUEST;
-    m.piece = req.piece;
+    m.piece = req.getPiece();
     m.begin = req.off;
     m.length = req.len;
     addMessage(m);
@@ -492,7 +492,7 @@ class PeerConnectionOut implements Runnable
           {
             Message m = (Message)it.next();
             if (m.type == Message.REQUEST
-                && m.piece == req.piece
+                && m.piece == req.getPiece()
                 && m.begin == req.off
                 && m.length == req.len)
               it.remove();
@@ -502,7 +502,7 @@ class PeerConnectionOut implements Runnable
     // Always send, just to be sure it it is really canceled.
     Message m = new Message();
     m.type = Message.CANCEL;
-    m.piece = req.piece;
+    m.piece = req.getPiece();
     m.begin = req.off;
     m.length = req.len;
     addMessage(m);

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java

@@ -22,6 +22,7 @@ package org.klomp.snark;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
@@ -48,7 +49,7 @@ import org.klomp.snark.dht.DHT;
 /**
  * Coordinates what peer does what.
  */
-public class PeerCoordinator implements PeerListener
+class PeerCoordinator implements PeerListener
 {
   private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerCoordinator.class);
 
@@ -68,6 +69,7 @@ public class PeerCoordinator implements PeerListener
   // package local for access by CheckDownLoadersTask
   final static long CHECK_PERIOD = 40*1000; // 40 seconds
   final static int MAX_UPLOADERS = 6;
+  public static final long MAX_INACTIVE = 8*60*1000;
 
   /**
    * Approximation of the number of current uploaders.
@@ -116,15 +118,21 @@ public class PeerCoordinator implements PeerListener
    */
   private final List<Piece> wantedPieces;
 
-  /** partial pieces - lock by synching on wantedPieces */
+  /** The total number of bytes in wantedPieces, or -1 if not yet known.
+   *  Sync on wantedPieces.
+   *  @since 0.9.1
+   */
+  private long wantedBytes;
+
+  /** partial pieces - lock by synching on wantedPieces - TODO store Requests, not PartialPieces */
   private final List<PartialPiece> partialPieces;
 
-  private boolean halted = false;
+  private volatile boolean halted;
 
   private final MagnetState magnetState;
   private final CoordinatorListener listener;
   private final I2PSnarkUtil _util;
-  private static final Random _random = I2PAppContext.getGlobalContext().random();
+  private final Random _random;
   
   /**
    *  @param metainfo null if in magnet mode
@@ -134,6 +142,7 @@ public class PeerCoordinator implements PeerListener
                          CoordinatorListener listener, Snark torrent)
   {
     _util = util;
+    _random = util.getContext().random();
     this.id = id;
     this.infohash = infohash;
     this.metainfo = metainfo;
@@ -151,7 +160,7 @@ public class PeerCoordinator implements PeerListener
     // Install a timer to check the uploaders.
     // Randomize the first start time so multiple tasks are spread out,
     // this will help the behavior with global limits
-    timer = new CheckEvent(new PeerCheckerTask(_util, this));
+    timer = new CheckEvent(_util.getContext(), new PeerCheckerTask(_util, this));
     timer.schedule((CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD));
   }
   
@@ -161,8 +170,8 @@ public class PeerCoordinator implements PeerListener
    */
   private static class CheckEvent extends SimpleTimer2.TimedEvent {
       private final PeerCheckerTask _task;
-      public CheckEvent(PeerCheckerTask task) {
-          super(SimpleTimer2.getInstance());
+      public CheckEvent(I2PAppContext ctx, PeerCheckerTask task) {
+          super(ctx.simpleTimer2());
           _task = task;
       }
       public void timeReached() {
@@ -171,16 +180,22 @@ public class PeerCoordinator implements PeerListener
       }
   }
 
-  // only called externally from Storage after the double-check fails
+  /**
+   * Only called externally from Storage after the double-check fails.
+   * Sets wantedBytes too.
+   */
   public void setWantedPieces()
   {
-    if (metainfo == null || storage == null)
+    if (metainfo == null || storage == null) {
+        wantedBytes = -1;
         return;
+    }
     // Make a list of pieces
       synchronized(wantedPieces) {
           wantedPieces.clear();
           BitField bitfield = storage.getBitField();
           int[] pri = storage.getPiecePriorities();
+          long count = 0;
           for (int i = 0; i < metainfo.getPieces(); i++) {
               // only add if we don't have and the priority is >= 0
               if ((!bitfield.get(i)) &&
@@ -189,8 +204,10 @@ public class PeerCoordinator implements PeerListener
                   if (pri != null)
                       p.setPriority(pri[i]);
                   wantedPieces.add(p);
+                  count += metainfo.getPieceLength(i);
               }
           }
+          wantedBytes = count;
           Collections.shuffle(wantedPieces, _random);
       }
   }
@@ -233,7 +250,9 @@ public class PeerCoordinator implements PeerListener
   }
 
   /**
-   * Returns how many bytes are still needed to get the complete file.
+   *  Bytes not yet in storage. Does NOT account for skipped files.
+   *  Not exact (does not adjust for last piece size).
+   * Returns how many bytes are still needed to get the complete torrent.
    * @return -1 if in magnet mode
    */
   public long getLeft()
@@ -244,6 +263,15 @@ public class PeerCoordinator implements PeerListener
     return ((long) storage.needed()) * metainfo.getPieceLength(0);
   }
 
+  /**
+   *  Bytes still wanted. DOES account for skipped files.
+   *  @return exact value. or -1 if no storage yet.
+   *  @since 0.9.1
+   */
+  public long getNeededLength() {
+      return wantedBytes;
+  }
+
   /**
    * Returns the total number of uploaded bytes of all peers.
    */
@@ -330,14 +358,32 @@ public class PeerCoordinator implements PeerListener
     return infohash;
   }
 
+  /**
+   *  Inbound.
+   *  Not halted, peers < max.
+   *  @since 0.9.1
+   */
   public boolean needPeers()
   {
         return !halted && peers.size() < getMaxConnections();
   }
+
+  /**
+   *  Outbound.
+   *  Not halted, peers < max, and need pieces.
+   *  @since 0.9.1
+   */
+  public boolean needOutboundPeers() {
+        //return wantedBytes != 0 && needPeers();
+        // minus one to make it a little easier for new peers to get in on large swarms
+        return wantedBytes != 0 && !halted && peers.size() < getMaxConnections() - 1;
+  }
   
   /**
-   *  Reduce max if huge pieces to keep from ooming when leeching
-   *  @return 512K: 16; 1M: 11; 2M: 6
+   *  Formerly used to
+   *  reduce max if huge pieces to keep from ooming when leeching
+   *  but now we don't
+   *  @return usually 16	
    */
   private int getMaxConnections() {
     if (metainfo == null)
@@ -347,13 +393,14 @@ public class PeerCoordinator implements PeerListener
         return 4;
     if (pieces <= 5)
         return 6;
-    int size = metainfo.getPieceLength(0);
+    //int size = metainfo.getPieceLength(0);
     int max = _util.getMaxConnections();
-    if (size <= 512*1024 || completed())
+    // Now that we use temp files, no memory concern
+    //if (size <= 512*1024 || completed())
       return max;
-    if (size <= 1024*1024)
-      return (max + max + 2) / 3;
-    return (max + 2) / 3;
+    //if (size <= 1024*1024)
+    //  return (max + max + 2) / 3;
+    //return (max + 2) / 3;
   }
 
   public boolean halted() { return halted; }
@@ -380,10 +427,27 @@ public class PeerCoordinator implements PeerListener
     }
     // delete any saved orphan partial piece
     synchronized (partialPieces) {
+        for (PartialPiece pp : partialPieces) {
+            pp.release();
+        }
         partialPieces.clear();
     }
   }
 
+  /**
+   *  @since 0.9.1
+   */
+  public void restart() {
+    halted = false;
+    synchronized (uploaded_old) {
+        Arrays.fill(uploaded_old, 0);
+    }
+    synchronized (downloaded_old) {
+        Arrays.fill(downloaded_old, 0);
+    }
+    timer.schedule((CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD));
+  }
+
   public void connected(Peer peer)
   { 
     if (halted)
@@ -396,7 +460,7 @@ public class PeerCoordinator implements PeerListener
     synchronized(peers)
       {
         Peer old = peerIDInList(peer.getPeerID(), peers);
-        if ( (old != null) && (old.getInactiveTime() > 8*60*1000) ) {
+        if ( (old != null) && (old.getInactiveTime() > MAX_INACTIVE) ) {
             // idle for 8 minutes, kill the old con (32KB/8min = 68B/sec minimum for one block)
             if (_log.shouldLog(Log.WARN))
               _log.warn("Remomving old peer: " + peer + ": " + old + ", inactive for " + old.getInactiveTime());
@@ -468,7 +532,10 @@ public class PeerCoordinator implements PeerListener
     return null;
   }
 
-// returns true if actual attempt to add peer occurs
+  /**
+   * Add peer (inbound or outbound)
+   * @return true if actual attempt to add peer occurs
+   */
   public boolean addPeer(final Peer peer)
   {
     if (halted)
@@ -487,7 +554,7 @@ public class PeerCoordinator implements PeerListener
         need_more = (!peer.isConnected()) && peersize < getMaxConnections();
         // Check if we already have this peer before we build the connection
         Peer old = peerIDInList(peer.getPeerID(), peers);
-        need_more = need_more && ((old == null) || (old.getInactiveTime() > 8*60*1000));
+        need_more = need_more && ((old == null) || (old.getInactiveTime() > MAX_INACTIVE));
       }
 
     if (need_more)
@@ -630,22 +697,23 @@ public class PeerCoordinator implements PeerListener
    * -1 if none of the given pieces are wanted.
    */
   public int wantPiece(Peer peer, BitField havePieces) {
-      return wantPiece(peer, havePieces, true);
+      Piece pc = wantPiece(peer, havePieces, true);
+      return pc != null ? pc.getId() : -1;
   }
 
   /**
    * Returns one of pieces in the given BitField that is still wanted or
-   * -1 if none of the given pieces are wanted.
+   * null if none of the given pieces are wanted.
    *
    * @param record if true, actually record in our data structures that we gave the
    *               request to this peer. If false, do not update the data structures.
    * @since 0.8.2
    */
-  private int wantPiece(Peer peer, BitField havePieces, boolean record) {
+  private Piece wantPiece(Peer peer, BitField havePieces, boolean record) {
     if (halted) {
       if (_log.shouldLog(Log.WARN))
           _log.warn("We don't want anything from the peer, as we are halted!  peer=" + peer);
-      return -1;
+      return null;
     }
 
     Piece piece = null;
@@ -680,7 +748,7 @@ public class PeerCoordinator implements PeerListener
             // If we do end game all the time, we generate lots of extra traffic
             // when the seeder is super-slow and all the peers are "caught up"
             if (wantedSize > END_GAME_THRESHOLD)
-                return -1;  // nothing to request and not in end game
+                return null;  // nothing to request and not in end game
             // let's not all get on the same piece
             // Even better would be to sort by number of requests
             if (record)
@@ -704,7 +772,7 @@ public class PeerCoordinator implements PeerListener
                     _log.warn("nothing to even rerequest from " + peer + ": requested = " + requested);
                 //  _log.warn("nothing to even rerequest from " + peer + ": requested = " + requested 
                 //            + " wanted = " + wantedPieces + " peerHas = " + havePieces);
-                return -1; //If we still can't find a piece we want, so be it.
+                return null; //If we still can't find a piece we want, so be it.
             } else {
                 // Should be a lot smarter here -
                 // share blocks rather than starting from 0 with each peer.
@@ -719,7 +787,7 @@ public class PeerCoordinator implements PeerListener
                 _log.info(peer + " is now requesting: piece " + piece + " priority " + piece.getPriority());
             piece.setRequested(peer, true);
         }
-        return piece.getId();
+        return piece;
       } // synch
   }
 
@@ -736,6 +804,7 @@ public class PeerCoordinator implements PeerListener
           _log.debug("Updated piece priorities called but no priorities to set?");
           return;
       }
+      List<Piece> toCancel = new ArrayList();
       synchronized(wantedPieces) {
           // Add incomplete and previously unwanted pieces to the list
           // Temp to avoid O(n**2)
@@ -749,6 +818,7 @@ public class PeerCoordinator implements PeerListener
                   if (!want.get(i)) {
                       Piece piece = new Piece(i);
                       wantedPieces.add(piece);
+                      wantedBytes += metainfo.getPieceLength(i);
                       // As connections are already up, new Pieces will
                       // not have their PeerID list populated, so do that.
                           for (Peer p : peers) {
@@ -770,23 +840,32 @@ public class PeerCoordinator implements PeerListener
                    p.setPriority(priority);
                } else {
                    iter.remove();
-                   // cancel all peers
-                       for (Peer peer : peers) {
-                           peer.cancel(p.getId());
-                       }
+                   toCancel.add(p);
+                   wantedBytes -= metainfo.getPieceLength(p.getId());
                }
           }
           if (_log.shouldLog(Log.DEBUG))
               _log.debug("Updated piece priorities, now wanted: " + wantedPieces);
           // if we added pieces, they will be in-order unless we shuffle
           Collections.shuffle(wantedPieces, _random);
+      }
 
-          // update request queues, in case we added wanted pieces
-          // and we were previously uninterested
-              for (Peer peer : peers) {
-                  peer.request();
+      // cancel outside of wantedPieces lock to avoid deadlocks
+      if (!toCancel.isEmpty()) {
+          // cancel all peers
+          for (Peer peer : peers) {
+              for (Piece p : toCancel) {
+                  peer.cancel(p.getId());
+              }
           }
       }
+
+      // ditto, avoid deadlocks
+      // update request queues, in case we added wanted pieces
+      // and we were previously uninterested
+      for (Peer peer : peers) {
+          peer.request();
+      }
   }
 
   /**
@@ -811,8 +890,10 @@ public class PeerCoordinator implements PeerListener
         snark.stopTorrent();
         String msg = "Error reading the storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
         _log.error(msg, ioe);
-        SnarkManager.instance().addMessage(msg);
-        SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+        if (listener != null) {
+            listener.addMessage(msg);
+            listener.addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+        }
         throw new RuntimeException(msg, ioe);
       }
   }
@@ -846,10 +927,11 @@ public class PeerCoordinator implements PeerListener
    *
    * @throws RuntimeException on IOE saving the piece
    */
-  public boolean gotPiece(Peer peer, int piece, byte[] bs)
+  public boolean gotPiece(Peer peer, PartialPiece pp)
   {
     if (metainfo == null || storage == null)
         return true;
+    int piece = pp.getPiece();
     if (halted) {
       _log.info("Got while-halted piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName());
       return true; // We don't actually care anymore.
@@ -872,7 +954,7 @@ public class PeerCoordinator implements PeerListener
         
         try
           {
-            if (storage.putPiece(piece, bs))
+            if (storage.putPiece(pp))
               {
                 if (_log.shouldLog(Log.INFO))
                     _log.info("Got valid piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName());
@@ -890,38 +972,51 @@ public class PeerCoordinator implements PeerListener
             snark.stopTorrent();
             String msg = "Error writing storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
             _log.error(msg, ioe);
-            SnarkManager.instance().addMessage(msg);
-            SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+            if (listener != null) {
+                listener.addMessage(msg);
+                listener.addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+            }
             throw new RuntimeException(msg, ioe);
           }
         wantedPieces.remove(p);
+        wantedBytes -= metainfo.getPieceLength(p.getId());
       }
 
     // just in case
     removePartialPiece(piece);
 
+    boolean done = wantedBytes <= 0;
+
     // Announce to the world we have it!
     // Disconnect from other seeders when we get the last piece
-        List<Peer> toDisconnect = new ArrayList(); 
-        Iterator<Peer> it = peers.iterator();
-        while (it.hasNext())
-          {
-            Peer p = it.next();
+    List<Peer> toDisconnect = done ? new ArrayList() : null; 
+    for (Peer p : peers) {
             if (p.isConnected())
               {
-                  if (completed() && p.isCompleted())
+                  if (done && p.isCompleted())
                       toDisconnect.add(p);
                   else
                       p.have(piece);
               }
-          }
-        it = toDisconnect.iterator();
-        while (it.hasNext())
-          {
-            Peer p = it.next();
+    }
+
+    if (done) {
+        for (Peer p : toDisconnect) {
             p.disconnect(true);
-          }
+        }
     
+        // put msg on the console if partial, since Storage won't do it
+        if (!completed())
+            snark.storageCompleted(storage);
+
+        synchronized (partialPieces) {
+            for (PartialPiece ppp : partialPieces) {
+                ppp.release();
+            }
+            partialPieces.clear();
+        }
+    }
+
     return true;
   }
 
@@ -998,17 +1093,24 @@ public class PeerCoordinator implements PeerListener
    *  Also mark the piece unrequested if this peer was the only one.
    *
    *  @param peer partials, must include the zero-offset (empty) ones too
+   *              No dup pieces, piece.setDownloaded() must be set
    *  @since 0.8.2
    */
-  public void savePartialPieces(Peer peer, List<PartialPiece> partials)
+  public void savePartialPieces(Peer peer, List<Request> partials)
   {
-      if (halted)
-          return;
       if (_log.shouldLog(Log.INFO))
           _log.info("Partials received from " + peer + ": " + partials);
+      if (halted || completed()) {
+          for (Request req : partials) {
+              PartialPiece pp = req.getPartialPiece();
+              pp.release();
+          }
+          return;
+      }
       synchronized(wantedPieces) {
-          for (PartialPiece pp : partials) {
-              if (pp.getDownloaded() > 0) {
+          for (Request req : partials) {
+              PartialPiece pp = req.getPartialPiece();
+              if (req.off > 0) {
                   // PartialPiece.equals() only compares piece number, which is what we want
                   int idx = partialPieces.indexOf(pp);
                   if (idx < 0) {
@@ -1017,10 +1119,12 @@ public class PeerCoordinator implements PeerListener
                           _log.info("Saving orphaned partial piece (new) " + pp);
                   } else if (idx >= 0 && pp.getDownloaded() > partialPieces.get(idx).getDownloaded()) {
                       // replace what's there now
+                      partialPieces.get(idx).release();
                       partialPieces.set(idx, pp);
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Saving orphaned partial piece (bigger) " + pp);
                   } else {
+                      pp.release();
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Discarding partial piece (not bigger)" + pp);
                   }
@@ -1029,10 +1133,14 @@ public class PeerCoordinator implements PeerListener
                       // sorts by remaining bytes, least first
                       Collections.sort(partialPieces);
                       PartialPiece gone = partialPieces.remove(max);
+                      gone.release();
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Discarding orphaned partial piece (list full)" + gone);
                   }
-              }  // else drop the empty partial piece
+              } else {
+                  // drop the empty partial piece
+                  pp.release();
+              }
               // synchs on wantedPieces...
               markUnrequested(peer, pp.getPiece());
           }
@@ -1058,10 +1166,18 @@ public class PeerCoordinator implements PeerListener
               PartialPiece pp = iter.next();
               int savedPiece = pp.getPiece();
               if (havePieces.get(savedPiece)) {
-                 iter.remove();
                  // this is just a double-check, it should be in there
+                 boolean skipped = false;
                  for(Piece piece : wantedPieces) {
                      if (piece.getId() == savedPiece) {
+                         if (peer.isCompleted() && piece.getPeerCount() > 1) {
+                             // Try to preserve rarest-first when we have only one seeder
+                             // by not preferring a partial piece that others have too
+                             // from a seeder
+                             skipped = true;
+                             break;
+                         }
+                         iter.remove();
                          piece.setRequested(peer, true);
                          if (_log.shouldLog(Log.INFO)) {
                              _log.info("Restoring orphaned partial piece " + pp +
@@ -1070,8 +1186,12 @@ public class PeerCoordinator implements PeerListener
                          return pp;
                       }
                   }
-                  if (_log.shouldLog(Log.WARN))
-                      _log.warn("Partial piece " + pp + " NOT in wantedPieces??");
+                  if (_log.shouldLog(Log.WARN)) {
+                      if (skipped)
+                          _log.warn("Partial piece " + pp + " with multiple peers skipped for seeder");
+                      else
+                          _log.warn("Partial piece " + pp + " NOT in wantedPieces??");
+                  }
               }
           }
           if (_log.shouldLog(Log.WARN) && !partialPieces.isEmpty())
@@ -1079,14 +1199,9 @@ public class PeerCoordinator implements PeerListener
       }
       // ...and this section turns this into the general move-requests-around code!
       // Temporary? So PeerState never calls wantPiece() directly for now...
-      int piece = wantPiece(peer, havePieces);
-      if (piece >= 0) {
-          try {
-              return new PartialPiece(piece, metainfo.getPieceLength(piece));
-          } catch (OutOfMemoryError oom) {
-              if (_log.shouldLog(Log.WARN))
-                  _log.warn("OOM creating new partial piece");
-          }
+      Piece piece = wantPiece(peer, havePieces, true);
+      if (piece != null) {
+          return new PartialPiece(piece, metainfo.getPieceLength(piece.getId()), _util.getTempDir());
       }
       if (_log.shouldLog(Log.DEBUG))
           _log.debug("We have no partial piece to return");
@@ -1121,7 +1236,7 @@ public class PeerCoordinator implements PeerListener
               }
           }
       }
-      return wantPiece(peer, havePieces, false) >= 0;
+      return wantPiece(peer, havePieces, false) != null;
   }
 
   /**
@@ -1176,6 +1291,7 @@ public class PeerCoordinator implements PeerListener
           }
       } else if (id == ExtensionHandler.ID_HANDSHAKE) {
           sendPeers(peer);
+          sendDHT(peer);
       }
   }
 
@@ -1186,6 +1302,8 @@ public class PeerCoordinator implements PeerListener
    *  @since 0.8.4
    */
   void sendPeers(Peer peer) {
+      if (metainfo != null && metainfo.isPrivate())
+          return;
       Map<String, BEValue> handshake = peer.getHandshakeMap();
       if (handshake == null)
           return;
@@ -1202,6 +1320,26 @@ public class PeerCoordinator implements PeerListener
       } catch (InvalidBEncodingException ibee) {}
   }
 
+  /**
+   *  Send a DHT message to the peer, if we both support DHT.
+   *  @since DHT
+   */
+  void sendDHT(Peer peer) {
+      DHT dht = _util.getDHT();
+      if (dht == null)
+          return;
+      Map<String, BEValue> handshake = peer.getHandshakeMap();
+      if (handshake == null)
+          return;
+      BEValue bev = handshake.get("m");
+      if (bev == null)
+          return;
+      try {
+          if (bev.getMap().get(ExtensionHandler.TYPE_DHT) != null)
+              ExtensionHandler.sendDHT(peer, dht.getPort(), dht.getRPort());
+      } catch (InvalidBEncodingException ibee) {}
+  }
+
   /**
    *  Sets the storage after transition out of magnet mode
    *  Snark calls this after we call gotMetaInfo()
@@ -1219,20 +1357,23 @@ public class PeerCoordinator implements PeerListener
   /**
    *  PeerListener callback
    *  Tell the DHT to ping it, this will get back the node info
+   *  @param rport must be port + 1
    *  @since 0.8.4
    */
-  public void gotPort(Peer peer, int port) {
+  public void gotPort(Peer peer, int port, int rport) {
       DHT dht = _util.getDHT();
-      if (dht != null)
+      if (dht != null &&
+          port > 0 && port < 65535 && rport == port + 1)
           dht.ping(peer.getDestination(), port);
   }
 
   /**
+   *  Get peers from PEX -
    *  PeerListener callback
    *  @since 0.8.4
    */
   public void gotPeers(Peer peer, List<PeerID> peers) {
-      if (completed() || !needPeers())
+      if (!needOutboundPeers())
           return;
       Destination myDest = _util.getMyDestination();
       if (myDest == null)
@@ -1294,5 +1435,13 @@ public class PeerCoordinator implements PeerListener
         return listener.overUpBWLimit(total * 1000 / CHECK_PERIOD);
     return false;
   }
+
+  /**
+   *  Convenience
+   *  @since 0.9.2
+   */
+  public I2PSnarkUtil getUtil() {
+      return _util;
+  }
 }
 

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinatorSet.java

@@ -1,9 +1,10 @@
 package org.klomp.snark;
 
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.Iterator;
-import java.util.Set;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.i2p.crypto.SHA1Hash;
 
 /**
  * Hmm, any guesses as to what this is?  Used by the multitorrent functionality
@@ -12,26 +13,28 @@ import java.util.Set;
  * from it there too)
  */
 public class PeerCoordinatorSet {
-    private final Set _coordinators;
+    private final Map<SHA1Hash, PeerCoordinator> _coordinators;
     
     public PeerCoordinatorSet() {
-        _coordinators = new HashSet();
+        _coordinators = new ConcurrentHashMap();
     }
-    
-    public Iterator iterator() {
-        synchronized (_coordinators) {
-            return new ArrayList(_coordinators).iterator();
-        }
+     
+    public Iterator<PeerCoordinator> iterator() {
+        return _coordinators.values().iterator();
     }
-    
+
     public void add(PeerCoordinator coordinator) {
-        synchronized (_coordinators) {
-            _coordinators.add(coordinator);
-        }
+        _coordinators.put(new SHA1Hash(coordinator.getInfoHash()), coordinator);
     }
+
     public void remove(PeerCoordinator coordinator) {
-        synchronized (_coordinators) {
-            _coordinators.remove(coordinator);
-        }
+        _coordinators.remove(new SHA1Hash(coordinator.getInfoHash()));
+    }
+
+    /**
+     *  @since 0.9.2
+     */
+    public PeerCoordinator get(byte[] infoHash) {
+        return _coordinators.get(new SHA1Hash(infoHash));
     }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerID.java

@@ -52,6 +52,7 @@ public class PeerID implements Comparable
   /** whether we have tried to get the dest from the hash - only do once */
   private boolean triedDestLookup;
   private final int hash;
+  private final I2PSnarkUtil util;
 
   public PeerID(byte[] id, Destination address)
   {
@@ -60,6 +61,7 @@ public class PeerID implements Comparable
     this.port = 6881;
     this.destHash = address.calculateHash().getData();
     hash = calculateHash();
+    util = null;
   }
 
   /**
@@ -93,13 +95,15 @@ public class PeerID implements Comparable
     port = 6881;
     this.destHash = address.calculateHash().getData();
     hash = calculateHash();
+    util = null;
   }
 
   /**
    * Creates a PeerID from a destHash
+   * @param util for eventual destination lookup
    * @since 0.8.1
    */
-  public PeerID(byte[] dest_hash) throws InvalidBEncodingException
+  public PeerID(byte[] dest_hash, I2PSnarkUtil util) throws InvalidBEncodingException
   {
     // id and address remain null
     port = 6881;
@@ -107,6 +111,7 @@ public class PeerID implements Comparable
         throw new InvalidBEncodingException("bad hash length");
     destHash = dest_hash;
     hash = DataHelper.hashCode(dest_hash);
+    this.util = util;
   }
 
   public byte[] getID()
@@ -131,7 +136,7 @@ public class PeerID implements Comparable
   {
     if (address == null && destHash != null && !triedDestLookup) {
         String b32 = Base32.encode(destHash) + ".b32.i2p";
-        address = I2PAppContext.getGlobalContext().namingService().lookup(b32);
+        address = util.getDestination(b32);
         triedDestLookup = true;
     }
     return address;

apps/i2psnark/java/src/org/klomp/snark/PeerListener.java

@@ -95,12 +95,11 @@ interface PeerListener
    * will be closed.
    *
    * @param peer the Peer that got the piece.
-   * @param piece the piece number received.
-   * @param bs the byte array containing the piece.
+   * @param piece the piece received.
    *
    * @return true when the bytes represent the piece, false otherwise.
    */
-  boolean gotPiece(Peer peer, int piece, byte[] bs);
+  boolean gotPiece(Peer peer, PartialPiece piece);
 
   /**
    * Called when the peer wants (part of) a piece from us. Only called
@@ -167,7 +166,7 @@ interface PeerListener
    * @param peer the peer
    * @since 0.8.2
    */
-  void savePartialPieces(Peer peer, List<PartialPiece> pcs);
+  void savePartialPieces(Peer peer, List<Request> pcs);
 
   /**
    * Called when a peer has connected and there may be a partially
@@ -191,13 +190,14 @@ interface PeerListener
   void gotExtension(Peer peer, int id, byte[] bs);
 
   /**
-   * Called when a port message is received.
+   * Called when a DHT port message is received.
    *
    * @param peer the Peer that got the message.
-   * @param port the port
+   * @param port the query port
+   * @param rport the response port
    * @since 0.8.4
    */
-  void gotPort(Peer peer, int port);
+  void gotPort(Peer peer, int port, int rport);
 
   /**
    * Called when peers are received via PEX
@@ -207,4 +207,10 @@ interface PeerListener
    * @since 0.8.4
    */
   void gotPeers(Peer peer, List<PeerID> pIDList);
+
+  /**
+   *  Convenience
+   *  @since 0.9.2
+   */
+  public I2PSnarkUtil getUtil();
 }

apps/i2psnark/java/src/org/klomp/snark/PeerState.java

@@ -20,13 +20,10 @@
 
 package org.klomp.snark;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
@@ -42,13 +39,13 @@ class PeerState implements DataLoader
 
   // Interesting and choking describes whether we are interested in or
   // are choking the other side.
-  boolean interesting = false;
-  boolean choking = true;
+  volatile boolean interesting;
+  volatile boolean choking = true;
 
   // Interested and choked describes whether the other side is
   // interested in us or choked us.
-  boolean interested = false;
-  boolean choked = true;
+  volatile boolean interested;
+  volatile boolean choked = true;
 
   /** the pieces the peer has */
   BitField bitfield;
@@ -110,7 +107,7 @@ class PeerState implements DataLoader
         // The only problem with returning the partials to the coordinator
         // is that chunks above a missing request are lost.
         // Future enhancements to PartialPiece could keep track of the holes.
-        List<PartialPiece> pcs = returnPartialPieces();
+        List<Request> pcs = returnPartialPieces();
         if (!pcs.isEmpty()) {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug(peer + " got choked, returning partial pieces to the PeerCoordinator: " + pcs);
@@ -234,8 +231,8 @@ class PeerState implements DataLoader
         return;
       }
 
-    if (_log.shouldLog(Log.INFO))
-      _log.info("Queueing (" + piece + ", " + begin + ", "
+    if (_log.shouldLog(Log.DEBUG))
+        _log.debug("Queueing (" + piece + ", " + begin + ", "
                 + length + ")" + " to " + peer);
 
     // don't load the data into mem now, let PeerConnectionOut do it
@@ -270,8 +267,8 @@ class PeerState implements DataLoader
         return null;
       }
 
-    if (_log.shouldLog(Log.INFO))
-      _log.info("Sending (" + piece + ", " + begin + ", "
+    if (_log.shouldLog(Log.DEBUG))
+        _log.debug("Sending (" + piece + ", " + begin + ", "
                 + length + ")" + " to " + peer);
     return pieceBytes;
   }
@@ -307,22 +304,23 @@ class PeerState implements DataLoader
 
     if (_log.shouldLog(Log.DEBUG))
       _log.debug("got end of Chunk("
-                  + req.piece + "," + req.off + "," + req.len + ") from "
+                  + req.getPiece() + "," + req.off + "," + req.len + ") from "
                   + peer);
 
     // Last chunk needed for this piece?
-    if (getFirstOutstandingRequest(req.piece) == -1)
+    // FIXME if priority changed to skip, we will think we're done when we aren't
+    if (getFirstOutstandingRequest(req.getPiece()) == -1)
       {
         // warning - may block here for a while
-        if (listener.gotPiece(peer, req.piece, req.bs))
+        if (listener.gotPiece(peer, req.getPartialPiece()))
           {
             if (_log.shouldLog(Log.DEBUG))
-              _log.debug("Got " + req.piece + ": " + peer);
+              _log.debug("Got " + req.getPiece() + ": " + peer);
           }
         else
           {
             if (_log.shouldLog(Log.WARN))
-              _log.warn("Got BAD " + req.piece + " from " + peer);
+              _log.warn("Got BAD " + req.getPiece() + " from " + peer);
           }
       }
 
@@ -338,7 +336,7 @@ class PeerState implements DataLoader
   synchronized private int getFirstOutstandingRequest(int piece)
    {
     for (int i = 0; i < outstandingRequests.size(); i++)
-      if (outstandingRequests.get(i).piece == piece)
+      if (outstandingRequests.get(i).getPiece() == piece)
         return i;
     return -1;
   }
@@ -374,7 +372,7 @@ class PeerState implements DataLoader
     synchronized(this)
       {
         req = outstandingRequests.get(r);
-        while (req.piece == piece && req.off != begin
+        while (req.getPiece() == piece && req.off != begin
                && r < outstandingRequests.size() - 1)
           {
             r++;
@@ -382,7 +380,7 @@ class PeerState implements DataLoader
           }
         
         // Something wrong?
-        if (req.piece != piece || req.off != begin || req.len != length)
+        if (req.getPiece() != piece || req.off != begin || req.len != length)
           {
             if (_log.shouldLog(Log.INFO))
               _log.info("Unrequested or unneeded 'piece: "
@@ -430,13 +428,13 @@ class PeerState implements DataLoader
       Request rv = null;
       int lowest = Integer.MAX_VALUE;
       for (Request r :  outstandingRequests) {
-          if (r.piece == piece && r.off < lowest) {
+          if (r.getPiece() == piece && r.off < lowest) {
               lowest = r.off;
               rv = r;
           }
       }
       if (pendingRequest != null &&
-          pendingRequest.piece == piece && pendingRequest.off < lowest)
+          pendingRequest.getPiece() == piece && pendingRequest.off < lowest)
           rv = pendingRequest;
 
       if (_log.shouldLog(Log.DEBUG))
@@ -450,14 +448,16 @@ class PeerState implements DataLoader
    *  @return List of PartialPieces, even those with an offset == 0, or empty list
    *  @since 0.8.2
    */
-  synchronized List<PartialPiece> returnPartialPieces()
+  synchronized List<Request> returnPartialPieces()
   {
       Set<Integer> pcs = getRequestedPieces();
-      List<PartialPiece> rv = new ArrayList(pcs.size());
+      List<Request> rv = new ArrayList(pcs.size());
       for (Integer p : pcs) {
           Request req = getLowestOutstandingRequest(p.intValue());
-          if (req != null)
-              rv.add(new PartialPiece(req));
+          if (req != null) {
+              req.getPartialPiece().setDownloaded(req.off);
+              rv.add(req);
+          }
       }
       outstandingRequests.clear();
       pendingRequest = null;
@@ -471,9 +471,9 @@ class PeerState implements DataLoader
   synchronized private Set<Integer> getRequestedPieces() {
       Set<Integer> rv = new HashSet(outstandingRequests.size() + 1);
       for (Request req : outstandingRequests) {
-          rv.add(Integer.valueOf(req.piece));
+          rv.add(Integer.valueOf(req.getPiece()));
       if (pendingRequest != null)
-          rv.add(Integer.valueOf(pendingRequest.piece));
+          rv.add(Integer.valueOf(pendingRequest.getPiece()));
       }
       return rv;
   }
@@ -489,6 +489,13 @@ class PeerState implements DataLoader
   /** @since 0.8.2 */
   void extensionMessage(int id, byte[] bs)
   {
+      if (metainfo != null && metainfo.isPrivate() &&
+          (id == ExtensionHandler.ID_METADATA || id == ExtensionHandler.ID_PEX)) {
+          // shouldn't get this since we didn't advertise it but they could send it anyway
+          if (_log.shouldLog(Log.WARN))
+              _log.warn("Private torrent, ignoring ext msg " + id);
+          return;
+      }
       ExtensionHandler.handleMessage(peer, listener, id, bs);
       // Peer coord will get metadata from MagnetState,
       // verify, and then call gotMetaInfo()
@@ -519,10 +526,14 @@ class PeerState implements DataLoader
           setInteresting(true);
   }
 
-  /** @since 0.8.4 */
+  /**
+   *  Unused
+   *  @since 0.8.4
+   */
   void portMessage(int port)
   {
-      listener.gotPort(peer, port);
+      // for compatibility with old DHT PORT message
+      listener.gotPort(peer, port, port + 1);
   }
 
   void unknownMessage(int type, byte[] bs)
@@ -567,14 +578,14 @@ class PeerState implements DataLoader
    * @since 0.8.1
    */
   synchronized void cancelPiece(int piece) {
-        if (lastRequest != null && lastRequest.piece == piece)
+        if (lastRequest != null && lastRequest.getPiece() == piece)
           lastRequest = null;
         
         Iterator<Request> it = outstandingRequests.iterator();
         while (it.hasNext())
           {
             Request req = it.next();
-            if (req.piece == piece)
+            if (req.getPiece() == piece)
               {
                 it.remove();
                 // Send cancel even when we are choked to make sure that it is
@@ -590,10 +601,10 @@ class PeerState implements DataLoader
    * @since 0.8.1
    */
   synchronized boolean isRequesting(int piece) {
-      if (pendingRequest != null && pendingRequest.piece == piece)
+      if (pendingRequest != null && pendingRequest.getPiece() == piece)
           return true;
       for (Request req : outstandingRequests) {
-          if (req.piece == piece)
+          if (req.getPiece() == piece)
               return true;
       }
       return false;
@@ -675,7 +686,7 @@ class PeerState implements DataLoader
           {
             int pieceLength;
             boolean isLastChunk;
-            pieceLength = metainfo.getPieceLength(lastRequest.piece);
+            pieceLength = metainfo.getPieceLength(lastRequest.getPiece());
             isLastChunk = lastRequest.off + lastRequest.len == pieceLength;
 
             // Last part of a piece?
@@ -683,14 +694,13 @@ class PeerState implements DataLoader
               more_pieces = requestNextPiece();
             else
               {
-                    int nextPiece = lastRequest.piece;
+                    PartialPiece nextPiece = lastRequest.getPartialPiece();
                     int nextBegin = lastRequest.off + PARTSIZE;
-                    byte[] bs = lastRequest.bs;
                     int maxLength = pieceLength - nextBegin;
                     int nextLength = maxLength > PARTSIZE ? PARTSIZE
                                                           : maxLength;
                     Request req
-                      = new Request(nextPiece, bs, nextBegin, nextLength);
+                      = new Request(nextPiece,nextBegin, nextLength);
                     outstandingRequests.add(req);
                     if (!choked)
                       out.sendRequest(req);
@@ -736,7 +746,7 @@ class PeerState implements DataLoader
         // what piece to give us next.
         int nextPiece = listener.wantPiece(peer, bitfield);
         if (nextPiece != -1
-            && (lastRequest == null || lastRequest.piece != nextPiece)) {
+            && (lastRequest == null || lastRequest.getPiece() != nextPiece)) {
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug(peer + " want piece " + nextPiece);
                 // Fail safe to make sure we are interested

apps/i2psnark/java/src/org/klomp/snark/Piece.java

@@ -57,6 +57,15 @@ class Piece implements Comparable {
     /** caller must synchronize */
     public boolean removePeer(Peer peer) { return this.peers.remove(peer.getPeerID()); }
 
+    /**
+     * How many peers have this piece?
+     * Caller must synchronize
+     * @since 0.9.1
+     */
+    public int getPeerCount() {
+        return this.peers.size();
+    } 
+    
     /** caller must synchronize */
     public boolean isRequested() {
         return this.requests != null && !this.requests.isEmpty();

apps/i2psnark/java/src/org/klomp/snark/Request.java

@@ -20,14 +20,16 @@
 
 package org.klomp.snark;
 
+import java.io.DataInputStream;
+import java.io.IOException;
+
 /**
  * Holds all information needed for a partial piece request.
  * This class should be used only by PeerState, PeerConnectionIn, and PeerConnectionOut.
  */
 class Request
 {
-  final int piece;
-  final byte[] bs;
+  private final PartialPiece piece;
   final int off;
   final int len;
   long sendTime;
@@ -36,26 +38,49 @@ class Request
    * Creates a new Request.
    *
    * @param piece Piece number requested.
-   * @param bs byte array where response should be stored.
    * @param off the offset in the array.
    * @param len the number of bytes requested.
    */
-  Request(int piece, byte[] bs, int off, int len)
+  Request(PartialPiece piece, int off, int len)
   {
+    // Sanity check
+    if (off < 0 || len <= 0 || off + len > piece.getLength())
+      throw new IndexOutOfBoundsException("Illegal Request " + toString());
+
     this.piece = piece;
-    this.bs = bs;
     this.off = off;
     this.len = len;
+  }
 
-    // Sanity check
-    if (piece < 0 || off < 0 || len <= 0 || off + len > bs.length)
-      throw new IndexOutOfBoundsException("Illegal Request " + toString());
+  /**
+   *  @since 0.9.1
+   */
+  public void read(DataInputStream din) throws IOException {
+      piece.read(din, off, len);
+  }
+
+  /**
+   *  The piece number this Request is for
+   *
+   *  @since 0.9.1
+   */
+  public int getPiece() {
+      return piece.getPiece();
+  }
+
+  /**
+   *  The PartialPiece this Request is for
+   *
+   *  @since 0.9.1
+   */
+  public PartialPiece getPartialPiece() {
+      return piece;
   }
 
     @Override
   public int hashCode()
   {
-    return piece ^ off ^ len;
+    return piece.getPiece() ^ off ^ len;
   }
 
     @Override
@@ -64,7 +89,7 @@ class Request
     if (o instanceof Request)
       {
         Request req = (Request)o;
-        return req.piece == piece && req.off == off && req.len == len;
+        return req.piece.equals(piece) && req.off == off && req.len == len;
       }
 
     return false;
@@ -73,6 +98,6 @@ class Request
     @Override
   public String toString()
   {
-    return "(" + piece + "," + off + "," + len + ")";
+    return "(" + piece.getPiece() + "," + off + "," + len + ")";
   }
 }

apps/i2psnark/java/src/org/klomp/snark/Snark.java

@@ -20,12 +20,10 @@
 
 package org.klomp.snark;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -37,6 +35,7 @@ import net.i2p.I2PAppContext;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.data.Destination;
 import net.i2p.util.I2PThread;
+import net.i2p.util.Log;
 
 /**
  * Main Snark program startup class.
@@ -49,34 +48,12 @@ public class Snark
   private final static int MIN_PORT = 6881;
   private final static int MAX_PORT = 6889;
 
-  // Error messages (non-fatal)
-  public final static int ERROR   = 1;
-
-  // Warning messages
-  public final static int WARNING = 2;
-
-  // Notices (peer level)
-  public final static int NOTICE  = 3;
-
-  // Info messages (protocol policy level)
-  public final static int INFO    = 4;
-
-  // Debug info (protocol level)
-  public final static int DEBUG   = 5;
-
-  // Very low level stuff (network level)
-  public final static int ALL     = 6;
-
-  /**
-   * What level of debug info to show.
-   */
-  //public static int debug = NOTICE;
-
   // Whether or not to ask the user for commands while sharing
-  private static boolean command_interpreter = true;
+  //private static boolean command_interpreter = true;
 
   private static final String newline = System.getProperty("line.separator");
 
+/****
   private static final String copyright =
   "The Hunting of the Snark Project - Copyright (C) 2003 Mark J. Wielaard"
   + newline + newline
@@ -92,10 +69,12 @@ public class Snark
   "Press return for help. Type \"quit\" and return to stop.";
   private static final String help =
   "Commands: 'info', 'list', 'quit'.";
+****/
 
   // String indicating main activity
   String activity = "Not started";
   
+/****
   private static class OOMListener implements I2PThread.OOMEventListener {
       public void outOfMemory(OutOfMemoryError err) {
           try {
@@ -108,6 +87,7 @@ public class Snark
       }
       
   }
+****/
   
 /******** No, not maintaining a command-line client
 
@@ -247,11 +227,13 @@ public class Snark
   private TrackerClient trackerclient;
   private String rootDataDir = ".";
   private final CompleteListener completeListener;
-  private boolean stopped;
+  private volatile boolean stopped;
+  private volatile boolean starting;
   private byte[] id;
-  private byte[] infoHash;
+  private final byte[] infoHash;
   private String additionalTrackerURL;
   private final I2PSnarkUtil _util;
+  private final Log _log;
   private final PeerCoordinatorSet _peerCoordinatorSet;
   private String trackerProblems;
   private int trackerSeenPeers;
@@ -305,6 +287,7 @@ public class Snark
 
     completeListener = complistener;
     _util = util;
+    _log = util.getContext().logManager().getLog(Snark.class);
     _peerCoordinatorSet = peerCoordinatorSet;
     acceptor = connectionAcceptor;
 
@@ -315,10 +298,9 @@ public class Snark
     activity = "Network setup";
 
     id = generateID();
-    debug("My peer id: " + PeerID.idencode(id), Snark.INFO);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("My peer id: " + PeerID.idencode(id));
 
-    int port;
-    IOException lastException = null;
 /*
  * Don't start a tunnel if the torrent isn't going to be started.
  * If we are starting,
@@ -339,6 +321,7 @@ public class Snark
     meta = null;
     File f = null;
     InputStream in = null;
+    byte[] x_infoHash = null;
     try
       {
         f = new File(torrent);
@@ -346,6 +329,8 @@ public class Snark
           in = new FileInputStream(f);
         else
           {
+         /**** No, we don't ever fetch a torrent file this way
+               and we don't want to block in the constructor
             activity = "Getting torrent";
             File torrentFile = _util.get(torrent, 3);
             if (torrentFile == null) {
@@ -355,9 +340,11 @@ public class Snark
                 torrentFile.deleteOnExit();
                 in = new FileInputStream(torrentFile);
             }
+         *****/
+             throw new IOException("not found");
           }
         meta = new MetaInfo(in);
-        infoHash = meta.getInfoHash();
+        x_infoHash = meta.getInfoHash();
       }
     catch(IOException ioe)
       {
@@ -398,7 +385,9 @@ public class Snark
               try { in.close(); } catch (IOException ioe) {}
       }    
 
-    debug(meta.toString(), INFO);
+    infoHash = x_infoHash;  // final
+    if (_log.shouldLog(Log.INFO))
+        _log.info(meta.toString());
     
     // When the metainfo torrent was created from an existing file/dir
     // it already exists.
@@ -459,6 +448,7 @@ public class Snark
   {
     completeListener = complistener;
     _util = util;
+    _log = util.getContext().logManager().getLog(Snark.class);
     _peerCoordinatorSet = peerCoordinatorSet;
     acceptor = connectionAcceptor;
     this.torrent = torrent;
@@ -505,9 +495,19 @@ public class Snark
   }
 
   /**
-   * Start up contacting peers and querying the tracker
+   * Start up contacting peers and querying the tracker.
+   * Blocks if tunnel is not yet open.
    */
-  public void startTorrent() {
+  public synchronized void startTorrent() {
+      starting = true;
+      try {
+          x_startTorrent();
+      } finally {
+          starting = false;
+      }
+  }
+
+  private void x_startTorrent() {
     boolean ok = _util.connect();
     if (!ok) fatal("Unable to connect to I2P");
     if (coordinator == null) {
@@ -516,9 +516,11 @@ public class Snark
             fatal("Unable to listen for I2P connections");
         else {
             Destination d = serversocket.getManager().getSession().getMyDestination();
-            debug("Listening on I2P destination " + d.toBase64() + " / " + d.calculateHash().toBase64(), NOTICE);
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Listening on I2P destination " + d.toBase64() + " / " + d.calculateHash().toBase64());
         }
-        debug("Starting PeerCoordinator, ConnectionAcceptor, and TrackerClient", NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Starting PeerCoordinator, ConnectionAcceptor, and TrackerClient");
         activity = "Collecting pieces";
         coordinator = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this);
         if (_peerCoordinatorSet != null) {
@@ -538,21 +540,14 @@ public class Snark
     }
 
     stopped = false;
-    boolean coordinatorChanged = false;
     if (coordinator.halted()) {
-        // ok, we have already started and stopped, but the coordinator seems a bit annoying to
-        // restart safely, so lets build a new one to replace the old
+        coordinator.restart();
         if (_peerCoordinatorSet != null)
-            _peerCoordinatorSet.remove(coordinator);
-        PeerCoordinator newCoord = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this);
-        if (_peerCoordinatorSet != null)
-            _peerCoordinatorSet.add(newCoord);
-        coordinator = newCoord;
-        coordinatorChanged = true;
+            _peerCoordinatorSet.add(coordinator);
     }
-    if (!trackerclient.started() && !coordinatorChanged) {
+    if (!trackerclient.started()) {
         trackerclient.start();
-    } else if (trackerclient.halted() || coordinatorChanged) {
+    } else if (trackerclient.halted()) {
         if (storage != null) {
             try {
                  storage.reopen(rootDataDir);
@@ -563,23 +558,30 @@ public class Snark
                  fatal("Could not reopen storage", ioe);
              }
         }
-        TrackerClient newClient = new TrackerClient(_util, meta, additionalTrackerURL, coordinator, this);
-        if (!trackerclient.halted())
-            trackerclient.halt();
-        trackerclient = newClient;
         trackerclient.start();
     } else {
-        debug("NOT starting TrackerClient???", NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("NOT starting TrackerClient???");
     }
   }
+
   /**
    * Stop contacting the tracker and talking with peers
    */
   public void stopTorrent() {
+      stopTorrent(false);
+  }
+
+  /**
+   * Stop contacting the tracker and talking with peers
+   * @param fast if true, limit the life of the unannounce threads
+   * @since 0.9.1
+   */
+  public synchronized void stopTorrent(boolean fast) {
     stopped = true;
     TrackerClient tc = trackerclient;
     if (tc != null)
-        tc.halt();
+        tc.halt(fast);
     PeerCoordinator pc = coordinator;
     if (pc != null)
         pc.halt();
@@ -601,10 +603,12 @@ public class Snark
         _util.disconnect();
   }
 
+/****
   private static Snark parseArguments(String[] args)
   {
     return parseArguments(args, null, null);
   }
+****/
 
     // Accessors
 
@@ -668,6 +672,22 @@ public class Snark
         return stopped;
     }
 
+    /**
+     *  Startup in progress.
+     *  @since 0.9.1
+     */
+    public boolean isStarting() {
+        return starting && stopped;
+    }
+
+    /**
+     *  Set startup in progress.
+     *  @since 0.9.1
+     */
+    public void setStarting() {
+        starting = true;
+    }
+
     /**
      *  @since 0.8.4
      */
@@ -780,6 +800,41 @@ public class Snark
     }
 
     /**
+     *  Bytes not yet in storage. Does NOT account for skipped files.
+     *  @return exact value. or -1 if no storage yet.
+     *          getNeeded() * pieceLength(0) isn't accurate if last piece
+     *          is still needed.
+     *  @since 0.8.9
+     */
+    public long getRemainingLength() {
+        if (meta != null && storage != null) {
+            long needed = storage.needed();
+            long length0 = meta.getPieceLength(0);
+            long remaining = needed * length0;
+            // fixup if last piece is needed
+            int last = meta.getPieces() - 1;
+            if (last != 0 && !storage.getBitField().get(last))
+                remaining -= length0 - meta.getPieceLength(last);
+            return remaining;
+        }
+        return -1;
+    }
+
+    /**
+     *  Bytes still wanted. DOES account for skipped files.
+     *  FIXME -1 when not running.
+     *  @return exact value. or -1 if no storage yet or when not running.
+     *  @since 0.9.1
+     */
+    public long getNeededLength() {
+        PeerCoordinator coord = coordinator;
+        if (coord != null)
+            return coord.getNeededLength();
+        return -1;
+    }
+
+    /**
+     *  Does not account for skipped files.
      *  @return number of pieces still needed (magnet mode or not), or -1 if unknown
      *  @since 0.8.4
      */
@@ -891,7 +946,7 @@ public class Snark
           }
         else if (args[i].equals("--no-commands"))
           {
-            command_interpreter = false;
+            //command_interpreter = false;
             i++;
           }
         //else if (args[i].equals("--eepproxy"))
@@ -950,22 +1005,13 @@ public class Snark
   private static void usage()
   {
     System.out.println
-      ("Usage: snark [--debug [level]] [--no-commands] [--port <port>]");
+      ("Usage: snark [--no-commands] [--port <port>]");
     System.out.println
       ("             [--eepproxy hostname portnum]");
     System.out.println
       ("             [--i2cp routerHost routerPort ['name=val name=val name=val']]");
     System.out.println
       ("             (<url>|<file>)");
-    System.out.println
-      ("  --debug\tShows some extra info and stacktraces");
-    System.out.println
-      ("    level\tHow much debug details to show");
-    System.out.println
-      ("         \t(defaults to "
-       + NOTICE + ", with --debug to "
-       + INFO + ", highest level is "
-       + ALL + ").");
     System.out.println
       ("  --no-commands\tDon't read interactive commands or show usage info.");
     System.out.println
@@ -1004,22 +1050,18 @@ public class Snark
    */
   private void fatal(String s, Throwable t)
   {
-    _util.debug(s, ERROR, t);
+    _log.error(s, t);
     //System.err.println("snark: " + s + ((t == null) ? "" : (": " + t)));
     //if (debug >= INFO && t != null)
     //  t.printStackTrace();
     stopTorrent();
+    if (t != null)
+        s += ": " + t;
+    if (completeListener != null)
+        completeListener.fatal(this, s);
     throw new RuntimeException(s, t);
   }
 
-  /**
-   * Show debug info if debug is true.
-   */
-  private void debug(String s, int level)
-  {
-    _util.debug(s, level, null);
-  }
-
   /** CoordinatorListener - this does nothing */
   public void peerChange(PeerCoordinator coordinator, Peer peer)
   {
@@ -1036,10 +1078,12 @@ public class Snark
    * @since 0.8.4
    */
   public void gotMetaInfo(PeerCoordinator coordinator, MetaInfo metainfo) {
-      meta = metainfo;
       try {
-          storage = new Storage(_util, meta, this);
+          // The following two may throw IOE...
+          storage = new Storage(_util, metainfo, this);
           storage.check(rootDataDir);
+          // ... so don't set meta until here
+          meta = metainfo;
           if (completeListener != null) {
               String newName = completeListener.gotMetaInfo(this);
               if (newName != null)
@@ -1055,7 +1099,7 @@ public class Snark
       }
   }
 
-  private boolean allocating = false;
+  //private boolean allocating = false;
   public void storageCreateFile(Storage storage, String name, long length)
   {
     //if (allocating)
@@ -1063,7 +1107,7 @@ public class Snark
 
     //System.out.print("Creating file '" + name
     //                 + "' of length " + length + ": ");
-    allocating = true;
+    //allocating = true;
   }
 
   // How much storage space has been allocated
@@ -1071,9 +1115,9 @@ public class Snark
 
   public void storageAllocated(Storage storage, long length)
   {
-    allocating = true;
+    //allocating = true;
     //System.out.print(".");
-    allocated += length;
+    //allocated += length;
     //if (allocated == meta.getTotalLength())
     //  System.out.println(); // We have all the disk space we need.
   }
@@ -1083,7 +1127,7 @@ public class Snark
   private boolean prechecking = true;
   public void storageChecked(Storage storage, int num, boolean checked)
   {
-    allocating = false;
+    //allocating = false;
     if (!allChecked && !checking)
       {
         // Use the MetaInfo from the storage since our own might not
@@ -1095,9 +1139,10 @@ public class Snark
         //                   + " pieces: ");
         checking = true;
       }
-    if (!checking)
-      debug("Got " + (checked ? "" : "BAD ") + "piece: " + num,
-                  Snark.INFO);
+    if (!checking) {
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Got " + (checked ? "" : "BAD ") + "piece: " + num);
+    }
   }
 
   public void storageAllChecked(Storage storage)
@@ -1113,7 +1158,8 @@ public class Snark
   
   public void storageCompleted(Storage storage)
   {
-    debug("Completely received " + torrent, Snark.INFO);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Completely received " + torrent);
     //storage.close();
     //System.out.println("Completely received: " + torrent);
     if (completeListener != null)
@@ -1133,6 +1179,15 @@ public class Snark
     //System.exit(0);
   }
   
+  /**
+   * StorageListener and CoordinatorListener callback
+   * @since 0.9.2
+   */
+  public void addMessage(String message) {
+      if (completeListener != null)
+          completeListener.addMessage(this, message);
+  }
+
   public interface CompleteListener {
     public void torrentComplete(Snark snark);
     public void updateStatus(Snark snark);
@@ -1147,6 +1202,16 @@ public class Snark
      */
     public String gotMetaInfo(Snark snark);
 
+    /**
+     * @since 0.9
+     */
+    public void fatal(Snark snark, String error);
+
+    /**
+     * @since 0.9.2
+     */
+    public void addMessage(Snark snark, String message);
+
     // not really listeners but the easiest way to get back to an optional SnarkManager
     public long getSavedTorrentTime(Snark snark);
     public BitField getSavedTorrentBitField(Snark snark);
@@ -1161,8 +1226,8 @@ public class Snark
     if (_peerCoordinatorSet == null || uploaders <= 0)
       return false;
     int totalUploaders = 0;
-    for (Iterator iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
-      PeerCoordinator c = (PeerCoordinator)iter.next();
+    for (Iterator<PeerCoordinator> iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
+      PeerCoordinator c = iter.next();
       if (!c.halted())
         totalUploaders += c.uploaders;
     }
@@ -1175,13 +1240,14 @@ public class Snark
     if (_peerCoordinatorSet == null)
       return false;
     long total = 0;
-    for (Iterator iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
-      PeerCoordinator c = (PeerCoordinator)iter.next();
+    for (Iterator<PeerCoordinator> iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
+      PeerCoordinator c = iter.next();
       if (!c.halted())
         total += c.getCurrentUploadRate();
     }
     long limit = 1024l * _util.getMaxUpBW();
-    debug("Total up bw: " + total + " Limit: " + limit, Snark.WARNING);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Total up bw: " + total + " Limit: " + limit);
     return total > limit;
   }
 

apps/i2psnark/java/src/org/klomp/snark/SnarkShutdown.java

@@ -26,6 +26,7 @@ import net.i2p.util.I2PAppThread;
 
 /**
  * Makes sure everything ends correctly when shutting down.
+ * @deprecated unused
  */
 public class SnarkShutdown extends I2PAppThread
 {
@@ -61,7 +62,7 @@ public class SnarkShutdown extends I2PAppThread
 
     //Snark.debug("Halting TrackerClient...", Snark.INFO);
     if (trackerclient != null)
-      trackerclient.halt();
+      trackerclient.halt(true);
 
     //Snark.debug("Halting PeerCoordinator...", Snark.INFO);
     if (coordinator != null)

apps/i2psnark/java/src/org/klomp/snark/Storage.java

@@ -23,13 +23,18 @@ package org.klomp.snark;
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
 import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.StringTokenizer;
+import java.util.concurrent.ConcurrentHashMap;
 
 import net.i2p.crypto.SHA1;
+import net.i2p.util.Log;
 import net.i2p.util.SecureFile;
 
 /**
@@ -46,9 +51,12 @@ public class Storage
   private File[] RAFfile;    // File to make it easier to reopen
   /** priorities by file; default 0; may be null. @since 0.8.1 */
   private int[] priorities;
+  /** is the file empty and sparse? */
+  private boolean[] isSparse;
 
   private final StorageListener listener;
-  private I2PSnarkUtil _util;
+  private final I2PSnarkUtil _util;
+  private final Log _log;
 
   private /* FIXME final FIXME */ BitField bitfield; // BitField to represent the pieces
   private int needed; // Number of pieces needed
@@ -67,6 +75,10 @@ public class Storage
   public static final int MAX_PIECES = 10*1024;
   public static final long MAX_TOTAL_SIZE = MAX_PIECE_SIZE * (long) MAX_PIECES;
 
+  private static final Map<String, String> _filterNameCache = new ConcurrentHashMap();
+
+  private static final boolean _isWindows = System.getProperty("os.name").startsWith("Win");
+
   /**
    * Creates a new storage based on the supplied MetaInfo.  This will
    * try to create and/or check all needed files in the MetaInfo.
@@ -77,6 +89,7 @@ public class Storage
     throws IOException
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(Storage.class);
     this.metainfo = metainfo;
     this.listener = listener;
     needed = metainfo.getPieces();
@@ -95,10 +108,12 @@ public class Storage
    * @param announce may be null
    * @param listener may be null
    */
-  public Storage(I2PSnarkUtil util, File baseFile, String announce, StorageListener listener)
+  public Storage(I2PSnarkUtil util, File baseFile, String announce,
+                 boolean privateTorrent, StorageListener listener)
     throws IOException
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(Storage.class);
     this.listener = listener;
     // Create names, rafs and lengths arrays.
     getFiles(baseFile);
@@ -151,7 +166,7 @@ public class Storage
 
     byte[] piece_hashes = fast_digestCreate();
     metainfo = new MetaInfo(announce, baseFile.getName(), null, files,
-                            lengthsList, piece_size, piece_hashes, total);
+                            lengthsList, piece_size, piece_hashes, total, privateTorrent);
 
   }
 
@@ -195,7 +210,7 @@ public class Storage
     RAFtime = new long[size];
     RAFfile = new File[size];
     priorities = new int[size];
-
+    isSparse = new boolean[size];
 
     int i = 0;
     Iterator it = files.iterator();
@@ -221,8 +236,9 @@ public class Storage
         File[] files = f.listFiles();
         if (files == null)
           {
-            _util.debug("WARNING: Skipping '" + f 
-                        + "' not a normal file.", Snark.WARNING);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("WARNING: Skipping '" + f 
+                        + "' not a normal file.");
             return;
           }
         for (int i = 0; i < files.length; i++)
@@ -327,7 +343,8 @@ public class Storage
   }
 
   /**
-   *  Must call setPiecePriorities() after calling this
+   *  Must call Snark.updatePiecePriorities()
+   *  (which calls getPiecePriorities()) after calling this.
    *  @param file canonical path (non-directory)
    *  @param pri default 0; <0 to disable
    *  @since 0.8.1
@@ -433,14 +450,20 @@ public class Storage
   /** use a saved bitfield and timestamp from a config file */
   public void check(String rootDir, long savedTime, BitField savedBitField) throws IOException
   {
-    File base = new SecureFile(rootDir, filterName(metainfo.getName()));
+    File base;
+    boolean areFilesPublic = _util.getFilesPublic();
+    if (areFilesPublic)
+        base = new File(rootDir, filterName(metainfo.getName()));
+    else
+        base = new SecureFile(rootDir, filterName(metainfo.getName()));
     boolean useSavedBitField = savedTime > 0 && savedBitField != null;
 
     List<List<String>> files = metainfo.getFiles();
     if (files == null)
       {
         // Create base as file.
-        _util.debug("Creating/Checking file: " + base, Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Creating/Checking file: " + base);
         if (!base.createNewFile() && !base.exists())
           throw new IOException("Could not create file " + base);
 
@@ -450,6 +473,7 @@ public class Storage
         RAFlock = new Object[1];
         RAFtime = new long[1];
         RAFfile = new File[1];
+        isSparse = new boolean[1];
         lengths[0] = metainfo.getTotalLength();
         RAFlock[0] = new Object();
         RAFfile[0] = base;
@@ -463,7 +487,8 @@ public class Storage
     else
       {
         // Create base as dir.
-        _util.debug("Creating/Checking directory: " + base, Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Creating/Checking directory: " + base);
         if (!base.mkdir() && !base.isDirectory())
           throw new IOException("Could not create directory " + base);
 
@@ -476,10 +501,11 @@ public class Storage
         RAFlock = new Object[size];
         RAFtime = new long[size];
         RAFfile = new File[size];
+        isSparse = new boolean[size];
         for (int i = 0; i < size; i++)
           {
             List<String> path = files.get(i);
-            File f = createFileFromNames(base, path);
+            File f = createFileFromNames(base, path, areFilesPublic);
             // dup file name check after filtering
             for (int j = 0; j < i; j++) {
                 if (f.equals(RAFfile[j])) {
@@ -495,7 +521,7 @@ public class Storage
                     else
                         lastPath = '_' + lastPath;
                     path.set(last, lastPath);
-                    f = createFileFromNames(base, path);
+                    f = createFileFromNames(base, path, areFilesPublic);
                     j = 0;
                 }
             }
@@ -521,19 +547,22 @@ public class Storage
       bitfield = savedBitField;
       needed = metainfo.getPieces() - bitfield.count();
       _probablyComplete = complete();
-      _util.debug("Found saved state and files unchanged, skipping check", Snark.NOTICE);
+      if (_log.shouldLog(Log.INFO))
+          _log.info("Found saved state and files unchanged, skipping check");
     } else {
       // the following sets the needed variable
       changed = true;
       checkCreateFiles(false);
     }
     if (complete()) {
-        _util.debug("Torrent is complete", Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Torrent is complete");
     } else {
         // fixme saved priorities
         if (files != null)
             priorities = new int[files.size()];
-        _util.debug("Still need " + needed + " out of " + metainfo.getPieces() + " pieces", Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Still need " + needed + " out of " + metainfo.getPieces() + " pieces");
     }
   }
 
@@ -558,25 +587,61 @@ public class Storage
         '<', '>', ':', '"', '/', '\\', '|', '?', '*',
         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
-        0x7f };
+        0x7f,
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+        // unicode newlines
+        0x2028, 0x2029
+     };
 
   /**
    * Removes 'suspicious' characters from the given file name.
    * http://msdn.microsoft.com/en-us/library/aa365247%28VS.85%29.aspx
+   * Then replace chars not supported in the charset.
+   *
+   * This is called frequently and it can be pretty slow so cache the result.
+   *
+   * TODO: If multiple files in the same torrent map to the same filter name,
+   * the whole torrent will blow up. Check at torrent creation?
    */
   public static String filterName(String name)
   {
-    if (name.equals(".") || name.equals(" "))
-        return "_";
-    String rv = name;
-    if (rv.startsWith("."))
-        rv = '_' + rv.substring(1);
-    if (rv.endsWith(".") || rv.endsWith(" "))
-        rv = rv.substring(0, rv.length() - 1) + '_';
-    for (int i = 0; i < ILLEGAL.length; i++) {
-        if (rv.indexOf(ILLEGAL[i]) >= 0)
-            rv = rv.replace(ILLEGAL[i], '_');
+    String rv = _filterNameCache.get(name);
+    if (rv != null)
+        return rv;
+    if (name.equals(".") || name.equals(" ")) {
+        rv = "_";
+    } else {
+        rv = name;
+        if (rv.startsWith("."))
+            rv = '_' + rv.substring(1);
+        if (rv.endsWith(".") || rv.endsWith(" "))
+            rv = rv.substring(0, rv.length() - 1) + '_';
+        for (int i = 0; i < ILLEGAL.length; i++) {
+            if (rv.indexOf(ILLEGAL[i]) >= 0)
+                rv = rv.replace(ILLEGAL[i], '_');
+        }
+        // Replace characters not supported in the charset
+        if (!Charset.defaultCharset().name().equals("UTF-8")) {
+            try {
+                CharsetEncoder enc = Charset.defaultCharset().newEncoder();
+                if (!enc.canEncode(rv)) {
+                    String repl = rv;
+                    for (int i = 0; i < rv.length(); i++) {
+                        char c = rv.charAt(i);
+                        if (!enc.canEncode(c))
+                            repl = repl.replace(c, '_');
+                    }
+                    rv = repl;
+                }
+            } catch (Exception ex) {
+                ex.printStackTrace();
+            }
+        }
     }
+    _filterNameCache.put(name, rv);
     return rv;
   }
 
@@ -585,7 +650,7 @@ public class Storage
    *  things going in the wrong place if there are duplicates
    *  in intermediate path elements after filtering.
    */
-  private static File createFileFromNames(File base, List<String> names) throws IOException
+  private static File createFileFromNames(File base, List<String> names, boolean areFilesPublic) throws IOException
   {
     File f = null;
     Iterator<String> it = names.iterator();
@@ -595,7 +660,10 @@ public class Storage
         if (it.hasNext())
           {
             // Another dir in the hierarchy.
-            f = new File(base, name);
+            if (areFilesPublic)
+                f = new File(base, name);
+            else
+                f = new SecureFile(base, name);
             if (!f.mkdir() && !f.isDirectory())
               throw new IOException("Could not create directory " + f);
             base = f;
@@ -603,7 +671,10 @@ public class Storage
         else
           {
             // The final element (file) in the hierarchy.
-            f = new SecureFile(base, name);
+            if (areFilesPublic)
+                f = new File(base, name);
+            else
+                f = new SecureFile(base, name);
             if (!f.createNewFile() && !f.exists())
               throw new IOException("Could not create file " + f);
           }
@@ -647,6 +718,7 @@ public class Storage
     }
 
     // Make sure all files are available and of correct length
+    // The files should all exist as they have been created with zero length by createFilesFromNames()
     for (int i = 0; i < rafs.length; i++)
       {
         long length = RAFfile[i].length();
@@ -668,9 +740,11 @@ public class Storage
         } else {
           String msg = "File '" + names[i] + "' exists, but has wrong length (expected " +
                        lengths[i] + " but found " + length + ") - repairing corruption";
-          SnarkManager.instance().addMessage(msg);
-          _util.debug(msg, Snark.ERROR);
+          if (listener != null)
+              listener.addMessage(msg);
+          _log.error(msg);
           changed = true;
+          resume = true;
           _probablyComplete = false; // to force RW
           synchronized(RAFlock[i]) {
               checkRAF(i);
@@ -744,34 +818,55 @@ public class Storage
     }
   }
 
-  /** this calls openRAF(); caller must synnchronize and call closeRAF() */
+  /**
+   *  This creates a (presumably) sparse file so that reads won't fail with IOE.
+   *  Sets isSparse[nr] = true. balloonFile(nr) should be called later to
+   *  defrag the file.
+   *
+   *  This calls openRAF(); caller must synchronize and call closeRAF().
+   */
   private void allocateFile(int nr) throws IOException
   {
     // caller synchronized
     openRAF(nr, false);  // RW
-    // XXX - Is this the best way to make sure we have enough space for
-    // the whole file?
+    long remaining = lengths[nr];
     if (listener != null)
-        listener.storageCreateFile(this, names[nr], lengths[nr]);
-    final int ZEROBLOCKSIZE = piece_size;
-    byte[] zeros;
-    try {
-        zeros = new byte[ZEROBLOCKSIZE];
-    } catch (OutOfMemoryError oom) {
-        throw new IOException(oom.toString());
-    }
-    int i;
-    for (i = 0; i < lengths[nr]/ZEROBLOCKSIZE; i++)
-      {
-        rafs[nr].write(zeros);
-        if (listener != null)
-          listener.storageAllocated(this, ZEROBLOCKSIZE);
-      }
-    int size = (int)(lengths[nr] - i*ZEROBLOCKSIZE);
-    rafs[nr].write(zeros, 0, size);
+        listener.storageCreateFile(this, names[nr], remaining);
+    rafs[nr].setLength(remaining);
+    // don't bother ballooning later on Windows since there is no sparse file support
+    // until JDK7 using the JSR-203 interface.
+    // RAF seeks/writes do not create sparse files.
+    // Windows will zero-fill up to the point of the write, which
+    // will make the file fairly unfragmented, on average, at least until
+    // near the end where it will get exponentially more fragmented.
+    if (!_isWindows)
+        isSparse[nr] = true;
     // caller will close rafs[nr]
     if (listener != null)
-      listener.storageAllocated(this, size);
+      listener.storageAllocated(this, lengths[nr]);
+  }
+
+  /**
+   *  This "balloons" the file with zeros to eliminate disk fragmentation.,
+   *  Overwrites the entire file with zeros. Sets isSparse[nr] = false.
+   *
+   *  Caller must synchronize and call checkRAF() or openRAF().
+   *  @since 0.9.1
+   */
+  private void balloonFile(int nr) throws IOException
+  {
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Ballooning " + nr + ": " + RAFfile[nr]);
+    long remaining = lengths[nr];
+    final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
+    byte[] zeros = new byte[ZEROBLOCKSIZE];
+    rafs[nr].seek(0);
+    while (remaining > 0) {
+        int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
+        rafs[nr].write(zeros, 0, size);
+        remaining -= size;
+    }
+    isSparse[nr] = false;
   }
 
 
@@ -792,7 +887,7 @@ public class Storage
             closeRAF(i);
           }
         } catch (IOException ioe) {
-            _util.debug("Error closing " + RAFfile[i], Snark.ERROR, ioe);
+            _log.error("Error closing " + RAFfile[i], ioe);
             // gobble gobble
         }
       }
@@ -813,7 +908,8 @@ public class Storage
     try {
       bs = new byte[len];
     } catch (OutOfMemoryError oom) {
-      _util.debug("Out of memory, can't honor request for piece " + piece, Snark.WARNING, oom);
+      if (_log.shouldLog(Log.WARN))
+          _log.warn("Out of memory, can't honor request for piece " + piece, oom);
       return null;
     }
     getUncheckedPiece(piece, bs, off, len);
@@ -827,54 +923,66 @@ public class Storage
    * matches), otherwise false.
    * @exception IOException when some storage related error occurs.
    */
-  public boolean putPiece(int piece, byte[] ba) throws IOException
+  public boolean putPiece(PartialPiece pp) throws IOException
   {
-    // First check if the piece is correct.
-    // Copy the array first to be paranoid.
-    byte[] bs = (byte[]) ba.clone();
-    int length = bs.length;
-    boolean correctHash = metainfo.checkPiece(piece, bs, 0, length);
-    if (listener != null)
-      listener.storageChecked(this, piece, correctHash);
-    if (!correctHash)
-      return false;
+      int piece = pp.getPiece();
+      try {
+          synchronized(bitfield) {
+              if (bitfield.get(piece))
+                  return true; // No need to store twice.
+          }
 
-    synchronized(bitfield)
-      {
-        if (bitfield.get(piece))
-          return true; // No need to store twice.
-      }
+          // TODO alternative - check hash on the fly as we write to the file,
+          // to save another I/O pass
+          boolean correctHash = metainfo.checkPiece(pp);
+          if (listener != null)
+            listener.storageChecked(this, piece, correctHash);
+          if (!correctHash) {
+              return false;
+          }
 
-    // Early typecast, avoid possibly overflowing a temp integer
-    long start = (long) piece * (long) piece_size;
-    int i = 0;
-    long raflen = lengths[i];
-    while (start > raflen)
-      {
-        i++;
-        start -= raflen;
-        raflen = lengths[i];
-      }
+          // Early typecast, avoid possibly overflowing a temp integer
+          long start = (long) piece * (long) piece_size;
+          int i = 0;
+          long raflen = lengths[i];
+          while (start > raflen) {
+              i++;
+              start -= raflen;
+              raflen = lengths[i];
+          }
     
-    int written = 0;
-    int off = 0;
-    while (written < length)
-      {
-        int need = length - written;
-        int len = (start + need < raflen) ? need : (int)(raflen - start);
-        synchronized(RAFlock[i])
-          {
-            checkRAF(i);
-            rafs[i].seek(start);
-            rafs[i].write(bs, off + written, len);
-          }
-        written += len;
-        if (need - len > 0)
-          {
-            i++;
-            raflen = lengths[i];
-            start = 0;
+          int written = 0;
+          int off = 0;
+          int length = metainfo.getPieceLength(piece);
+          while (written < length) {
+              int need = length - written;
+              int len = (start + need < raflen) ? need : (int)(raflen - start);
+              synchronized(RAFlock[i]) {
+                  checkRAF(i);
+                  if (isSparse[i]) {
+                      // If the file is a newly created sparse file,
+                      // AND we aren't skipping it, balloon it with all
+                      // zeros to un-sparse it by allocating the space.
+                      // Obviously this could take a while.
+                      // Once we have written to it, it isn't empty/sparse any more.
+                      if (priorities == null || priorities[i] >= 0)
+                          balloonFile(i);
+                      else
+                          isSparse[i] = false;
+                  }
+                  rafs[i].seek(start);
+                  //rafs[i].write(bs, off + written, len);
+                  pp.write(rafs[i], off + written, len);
+              }
+              written += len;
+              if (need - len > 0) {
+                  i++;
+                  raflen = lengths[i];
+                  start = 0;
+              }
           }
+      } finally {
+          pp.release();
       }
 
     changed = true;
@@ -905,13 +1013,14 @@ public class Storage
       if (needed > 0) {
         if (listener != null)
             listener.setWantedPieces(this);
-        _util.debug("WARNING: Not really done, missing " + needed
-                    + " pieces", Snark.WARNING);
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("WARNING: Not really done, missing " + needed
+                    + " pieces");
       }
     }
 
     return true;
-  }
+ }
 
   /**
    *  This is a dup of MetaInfo.getPieceLength() but we need it
@@ -976,7 +1085,7 @@ public class Storage
   /**
    * Close unused RAFs - call periodically
    */
-  private static final long RAFCloseDelay = 7*60*1000;
+  private static final long RAFCloseDelay = 4*60*1000;
   public void cleanRAFs() {
     long cutoff = System.currentTimeMillis() - RAFCloseDelay;
     for (int i = 0; i < RAFlock.length; i++) {

apps/i2psnark/java/src/org/klomp/snark/StorageListener.java

@@ -23,7 +23,7 @@ package org.klomp.snark;
 /**
  * Callback used when Storage changes.
  */
-public interface StorageListener
+interface StorageListener
 {
   /**
    * Called when the storage creates a new file of a given length.
@@ -61,4 +61,6 @@ public interface StorageListener
    *
    */
   void setWantedPieces(Storage storage);
+
+  void addMessage(String message);
 }

apps/i2psnark/java/src/org/klomp/snark/Tracker.java

@@ -0,0 +1,28 @@
+/*
+ * Released into the public domain 
+ * with no warranty of any kind, either expressed or implied.  
+ */
+package org.klomp.snark;
+
+/**
+ * A structure for known trackers
+ *
+ * @since 0.9.1
+ */
+public class Tracker {
+
+    public final String name;
+    public final String announceURL;
+    public final String baseURL;
+    public final boolean supportsDetails;
+
+    /**
+     *  @param baseURL The web site, may be null
+     */
+    public Tracker(String name, String announceURL, String baseURL) {
+        this.name = name;
+        this.announceURL = announceURL;
+        this.baseURL = baseURL;
+        this.supportsDetails = name.contains("tracker2.postman.i2p");
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java

@@ -27,16 +27,21 @@ import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Random;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer2;
 
 import org.klomp.snark.dht.DHT;
 
@@ -44,11 +49,22 @@ import org.klomp.snark.dht.DHT;
  * Informs metainfo tracker of events and gets new peers for peer
  * coordinator.
  *
+ * start() creates a thread and starts it.
+ * At the end of each run, a TimedEvent is queued on the SimpleTimer2 queue.
+ * The TimedEvent creates a new thread and starts it, so it does not
+ * clog SimpleTimer2.
+ *
+ * The thread runs one pass through the trackers, the PEX, and the DHT,
+ * then queues a new TimedEvent and exits.
+ *
+ * Thus there are only threads that are actively announcing, not one thread per torrent forever.
+ *
+ * start() may be called again after halt().
+ *
  * @author Mark Wielaard (mark@klomp.org)
  */
-public class TrackerClient extends I2PAppThread
-{
-  private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(TrackerClient.class);
+public class TrackerClient implements Runnable {
+  private final Log _log;
   private static final String NO_EVENT = "";
   private static final String STARTED_EVENT = "started";
   private static final String COMPLETED_EVENT = "completed";
@@ -57,25 +73,43 @@ public class TrackerClient extends I2PAppThread
 
   private final static int SLEEP = 5; // 5 minutes.
   private final static int DELAY_MIN = 2000; // 2 secs.
-  private final static int DELAY_MUL = 1500; // 1.5 secs.
+  private final static int DELAY_RAND = 6*1000;
   private final static int MAX_REGISTER_FAILS = 10; // * INITIAL_SLEEP = 15m to register
   private final static int INITIAL_SLEEP = 90*1000;
   private final static int MAX_CONSEC_FAILS = 5;    // slow down after this
   private final static int LONG_SLEEP = 30*60*1000; // sleep a while after lots of fails
+  private final static long MIN_TRACKER_ANNOUNCE_INTERVAL = 10*60*1000;
+  private final static long MIN_DHT_ANNOUNCE_INTERVAL = 10*60*1000;
 
-  private I2PSnarkUtil _util;
+  private final I2PSnarkUtil _util;
   private final MetaInfo meta;
+  private final String infoHash;
+  private final String peerID;
   private final String additionalTrackerURL;
   private final PeerCoordinator coordinator;
   private final Snark snark;
   private final int port;
+  private final String _threadName;
 
-  private boolean stop;
-  private boolean started;
-
-  private List trackers;
+  private volatile boolean stop = true;
+  private volatile boolean started;
+  private volatile boolean _initialized;
+  private volatile int _runCount;
+  // running thread so it can be interrupted
+  private volatile Thread _thread;
+  // queued event so it can be cancelled
+  private volatile SimpleTimer2.TimedEvent _event;
+  // these 2 used in loop()
+  private volatile boolean runStarted;
+  private volatile  int consecutiveFails;
+  private boolean completed;
+  private volatile boolean _fastUnannounce;
+  private long lastDHTAnnounce;
+  private final List<Tracker> trackers;
 
   /**
+   * Call start() to start it.
+   *
    * @param meta null if in magnet mode
    * @param additionalTrackerURL may be null, from the ?tr= param in magnet mode, otherwise ignored
    */
@@ -85,20 +119,32 @@ public class TrackerClient extends I2PAppThread
     super();
     // Set unique name.
     String id = urlencode(snark.getID());
-    setName("TrackerClient " + id.substring(id.length() - 12));
+    _threadName = "TrackerClient " + id.substring(id.length() - 12);
     _util = util;
+    _log = util.getContext().logManager().getLog(TrackerClient.class);
     this.meta = meta;
     this.additionalTrackerURL = additionalTrackerURL;
     this.coordinator = coordinator;
     this.snark = snark;
 
     this.port = 6881; //(port == -1) ? 9 : port;
+    this.infoHash = urlencode(snark.getInfoHash());
+    this.peerID = urlencode(snark.getID());
+    this.trackers = new ArrayList(2);
   }
 
-    @Override
-  public void start() {
-      if (stop) throw new RuntimeException("Dont rerun me, create a copy");
-      super.start();
+  public synchronized void start() {
+      if (!stop) {
+          if (_log.shouldLog(Log.WARN))
+              _log.warn("Already started: " + _threadName);
+          return;
+      }
+      stop = false;
+      consecutiveFails = 0;
+      runStarted = false;
+      _fastUnannounce = false;
+      _thread = new I2PAppThread(this, _threadName + " #" + (++_runCount), true);
+      _thread.start();
       started = true;
   }
   
@@ -107,11 +153,50 @@ public class TrackerClient extends I2PAppThread
   
   /**
    * Interrupts this Thread to stop it.
+   * @param fast if true, limit the life of the unannounce threads
    */
-  public void halt()
-  {
-    stop = true;
-    this.interrupt();
+  public synchronized void halt(boolean fast) {
+    boolean wasStopped = stop;
+    if (wasStopped) {
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("Already stopped: " + _threadName);
+    } else {
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("Stopping: " + _threadName);
+        stop = true;
+    }
+    SimpleTimer2.TimedEvent e = _event;
+    if (e != null) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Cancelling next announce " + _threadName);
+        e.cancel();
+        _event = null;
+    }
+    Thread t = _thread;
+    if (t != null) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Interrupting " + t.getName());
+        t.interrupt();
+    }
+    _fastUnannounce = true;
+    if (!wasStopped)
+        unannounce();
+  }
+
+  private void queueLoop(long delay) {
+      _event = new Runner(delay);
+  }
+
+  private class Runner extends SimpleTimer2.TimedEvent {
+      public Runner(long delay) {
+          super(_util.getContext().simpleTimer2(), delay);
+      }
+
+      public void timeReached() {
+          _event = null;
+          _thread = new I2PAppThread(TrackerClient.this, _threadName + " #" + (++_runCount), true);
+          _thread.start();
+      }
   }
 
   private boolean verifyConnected() {
@@ -124,20 +209,56 @@ public class TrackerClient extends I2PAppThread
     return !stop && _util.connected();
   }
   
-    @Override
-  public void run()
-  {
-    String infoHash = urlencode(snark.getInfoHash());
-    String peerID = urlencode(snark.getID());
-
+  /**
+   *  Setup the first time only,
+   *  then one pass (usually) through the trackers, PEX, and DHT.
+   *  This will take several seconds to several minutes.
+   */
+  public void run() {
+      long begin = _util.getContext().clock().now();
+      if (_log.shouldLog(Log.DEBUG))
+          _log.debug("Start " + Thread.currentThread().getName());
+      try {
+          if (!_initialized) {
+              setup();
+          }
+          if (trackers.isEmpty() && _util.getDHT() == null) {
+              stop = true;
+              this.snark.addMessage(_util.getString("No valid trackers for {0} - enable opentrackers or DHT?",
+                                              this.snark.getBaseName()));
+              _log.error("No valid trackers for " + this.snark.getBaseName());
+              this.snark.stopTorrent();
+              return;
+          }
+          if (!_initialized) {
+              _initialized = true;
+              // FIXME only when starting everybody at once, not for a single torrent
+              long delay = I2PAppContext.getGlobalContext().random().nextInt(30*1000);
+              try {
+                  Thread.sleep(delay);
+              } catch (InterruptedException ie) {}
+          }
+          loop();
+      } finally {
+          // don't hold ref
+          _thread = null;
+          if (_log.shouldLog(Log.DEBUG))
+              _log.debug("Finish " + Thread.currentThread().getName() +
+                         " after " + DataHelper.formatDuration(_util.getContext().clock().now() - begin));
+      }
+  }
 
+  /**
+   *  Do this one time only (not every time it is started).
+   *  @since 0.9.1
+   */
+  private void setup() {
     // Construct the list of trackers for this torrent,
     // starting with the primary one listed in the metainfo,
     // followed by the secondary open trackers
     // It's painful, but try to make sure if an open tracker is also
     // the primary tracker, that we don't add it twice.
     // todo: check for b32 matches as well
-    trackers = new ArrayList(2);
     String primary = null;
     if (meta != null)
         primary = meta.getAnnounce();
@@ -155,7 +276,7 @@ public class TrackerClient extends I2PAppThread
         primary = "";
     }
     List tlist = _util.getOpenTrackers();
-    if (tlist != null) {
+    if (tlist != null && (meta == null || !meta.isPrivate())) {
         for (int i = 0; i < tlist.size(); i++) {
              String url = (String)tlist.get(i);
              if (!isValidAnnounce(url)) {
@@ -183,70 +304,33 @@ public class TrackerClient extends I2PAppThread
              _log.debug("Additional announce: [" + url + "] for infoHash: " + infoHash);
         }
     }
+    this.completed = coordinator.getLeft() == 0;
+  }
 
-    if (trackers.isEmpty()) {
-        stop = true;
-        // FIXME translate
-        SnarkManager.instance().addMessage("No valid trackers for " + this.snark.getBaseName() + " - enable opentrackers?");
-        _log.error("No valid trackers for " + this.snark.getBaseName());
-        // FIXME keep going if DHT enabled
-        this.snark.stopTorrent();
-        return;
-    }
-
-    long uploaded = coordinator.getUploaded();
-    long downloaded = coordinator.getDownloaded();
-    long left = coordinator.getLeft();
-
-    boolean completed = (left == 0);
-    int sleptTime = 0;
-
+  /**
+   *  Announce to all the trackers, get peers from PEX and DHT, then queue up a SimpleTimer2 event.
+   *  This will take several seconds to several minutes.
+   *  @since 0.9.1
+   */
+  private void loop() {
     try
       {
-        if (!verifyConnected()) return;
-        boolean runStarted = false;
-        boolean firstTime = true;
-        int consecutiveFails = 0;
         Random r = I2PAppContext.getGlobalContext().random();
         while(!stop)
           {
+            if (!verifyConnected()) {
+                stop = true;
+                return;
+            }
+
             // Local DHT tracker announce
-            if (_util.getDHT() != null)
-                _util.getDHT().announce(snark.getInfoHash());
-            try
-              {
-                // Sleep some minutes...
-                // Sleep the minimum interval for all the trackers, but 60s minimum
-                // except for the first time...
-                int delay;
-                int random = r.nextInt(120*1000);
-                if (firstTime) {
-                  delay = r.nextInt(30*1000);
-                  firstTime = false;
-                } else if (completed && runStarted)
-                  delay = 3*SLEEP*60*1000 + random;
-                else if (snark.getTrackerProblems() != null && ++consecutiveFails < MAX_CONSEC_FAILS)
-                  delay = INITIAL_SLEEP;
-                else
-                  // sleep a while, when we wake up we will contact only the trackers whose intervals have passed
-                  delay = SLEEP*60*1000 + random;
+            DHT dht = _util.getDHT();
+            if (dht != null)
+                dht.announce(snark.getInfoHash());
 
-                if (delay > 0)
-                  Thread.sleep(delay);
-              }
-            catch(InterruptedException interrupt)
-              {
-                // ignore
-              }
-
-            if (stop)
-              break;
-       
-            if (!verifyConnected()) return;
-            
-            uploaded = coordinator.getUploaded();
-            downloaded = coordinator.getDownloaded();
-            left = coordinator.getLeft();   // -1 in magnet mode
+            long uploaded = coordinator.getUploaded();
+            long downloaded = coordinator.getDownloaded();
+            long left = coordinator.getLeft();   // -1 in magnet mode
             
             // First time we got a complete download?
             String event;
@@ -259,13 +343,10 @@ public class TrackerClient extends I2PAppThread
               event = NO_EVENT;
             
             // *** loop once for each tracker
-            // Only do a request when necessary.
-            sleptTime = 0;
             int maxSeenPeers = 0;
-            for (Iterator iter = trackers.iterator(); iter.hasNext(); ) {
-              Tracker tr = (Tracker)iter.next();
+            for (Tracker tr : trackers) {
               if ((!stop) && (!tr.stop) &&
-                  (completed || coordinator.needPeers()) &&
+                  (completed || coordinator.needOutboundPeers() || !tr.started) &&
                   (event.equals(COMPLETED_EVENT) || System.currentTimeMillis() > tr.lastRequestTime + tr.interval))
               {
                 try
@@ -291,25 +372,25 @@ public class TrackerClient extends I2PAppThread
                         snark.setTrackerSeenPeers(tr.seenPeers);
 
                     // pass everybody over to our tracker
-                    if (_util.getDHT() != null) {
+                    dht = _util.getDHT();
+                    if (dht != null) {
                         for (Peer peer : peers) {
-                            _util.getDHT().announce(snark.getInfoHash(), peer.getPeerID().getDestHash());
+                            dht.announce(snark.getInfoHash(), peer.getPeerID().getDestHash());
                         }
                     }
 
-                    if ( (left != 0) && (!completed) ) {
+                    if (coordinator.needOutboundPeers()) {
                         // we only want to talk to new people if we need things
                         // from them (duh)
                         List<Peer> ordered = new ArrayList(peers);
                         Collections.shuffle(ordered, r);
                         Iterator<Peer> it = ordered.iterator();
-                        while ((!stop) && it.hasNext()) {
+                        while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
                           Peer cur = it.next();
                           // FIXME if id == us || dest == us continue;
                           // only delay if we actually make an attempt to add peer
                           if(coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN;
-                            sleptTime += delay;
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
                             try { Thread.sleep(delay); } catch (InterruptedException ie) {}
                           }
                         }
@@ -318,14 +399,15 @@ public class TrackerClient extends I2PAppThread
                 catch (IOException ioe)
                   {
                     // Probably not fatal (if it doesn't last to long...)
-                    _util.debug
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn
                       ("WARNING: Could not contact tracker at '"
-                       + tr.announce + "': " + ioe, Snark.WARNING);
+                       + tr.announce + "': " + ioe);
                     tr.trackerProblems = ioe.getMessage();
                     // don't show secondary tracker problems to the user
                     if (tr.isPrimary)
                       snark.setTrackerProblems(tr.trackerProblems);
-                    if (tr.trackerProblems.toLowerCase().startsWith(NOT_REGISTERED)) {
+                    if (tr.trackerProblems.toLowerCase(Locale.US).startsWith(NOT_REGISTERED)) {
                       // Give a guy some time to register it if using opentrackers too
                       if (trackers.size() == 1) {
                         stop = true;
@@ -341,101 +423,184 @@ public class TrackerClient extends I2PAppThread
                             tr.interval = LONG_SLEEP;  // slow down
                     }
                   }
+              } else {
+                  if (_log.shouldLog(Log.INFO))
+                      _log.info("Not announcing to " + tr.announce + " last announce was " +
+                               new Date(tr.lastRequestTime) + " interval is " + DataHelper.formatDuration(tr.interval));
               }
               if ((!tr.stop) && maxSeenPeers < tr.seenPeers)
                   maxSeenPeers = tr.seenPeers;
             }  // *** end of trackers loop here
 
             // Get peers from PEX
-            if (left > 0 && coordinator.needPeers() && !stop) {
+            if (coordinator.needOutboundPeers() && (meta == null || !meta.isPrivate()) && !stop) {
                 Set<PeerID> pids = coordinator.getPEXPeers();
                 if (!pids.isEmpty()) {
-                    _util.debug("Got " + pids.size() + " from PEX", Snark.INFO);
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Got " + pids.size() + " from PEX");
                     List<Peer> peers = new ArrayList(pids.size());
                     for (PeerID pID : pids) {
                         peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
                     }
                     Collections.shuffle(peers, r);
                     Iterator<Peer> it = peers.iterator();
-                    while ((!stop) && it.hasNext()) {
+                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
                         Peer cur = it.next();
                         if (coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN;
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
                             try { Thread.sleep(delay); } catch (InterruptedException ie) {}
                          }
                     }
                 }
+            } else {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Not getting PEX peers");
             }
 
             // Get peers from DHT
             // FIXME this needs to be in its own thread
-            if (_util.getDHT() != null && !stop) {
+            dht = _util.getDHT();
+            if (dht != null && (meta == null || !meta.isPrivate()) && (!stop) &&
+                _util.getContext().clock().now() >  lastDHTAnnounce + MIN_DHT_ANNOUNCE_INTERVAL) {
                 int numwant;
-                if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers())
+                if (event.equals(STOPPED_EVENT) || !coordinator.needOutboundPeers())
                     numwant = 1;
                 else
                     numwant = _util.getMaxConnections();
-                List<Hash> hashes = _util.getDHT().getPeers(snark.getInfoHash(), numwant, 2*60*1000);
-                _util.debug("Got " + hashes + " from DHT", Snark.INFO);
+                Collection<Hash> hashes = dht.getPeers(snark.getInfoHash(), numwant, 2*60*1000);
+                if (!hashes.isEmpty()) {
+                    runStarted = true;
+                    lastDHTAnnounce = _util.getContext().clock().now();
+                }
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Got " + hashes + " from DHT");
                 // announce  ourselves while the token is still good
                 // FIXME this needs to be in its own thread
                 if (!stop) {
-                    int good = _util.getDHT().announce(snark.getInfoHash(), 8, 5*60*1000);
-                    _util.debug("Sent " + good + " good announces to DHT", Snark.INFO);
+                    // announce only to the 1 closest
+                    int good = dht.announce(snark.getInfoHash(), 1, 5*60*1000);
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Sent " + good + " good announces to DHT");
                 }
 
                 // now try these peers
                 if ((!stop) && !hashes.isEmpty()) {
                     List<Peer> peers = new ArrayList(hashes.size());
                     for (Hash h : hashes) {
-                        PeerID pID = new PeerID(h.getData());
+                        PeerID pID = new PeerID(h.getData(), _util);
                         peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
                     }
                     Collections.shuffle(peers, r);
                     Iterator<Peer> it = peers.iterator();
-                    while ((!stop) && it.hasNext()) {
+                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
                         Peer cur = it.next();
                         if (coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = (DELAY_MUL * r.nextInt(10)) + DELAY_MIN;
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
                             try { Thread.sleep(delay); } catch (InterruptedException ie) {}
                          }
                     }
                 }
+            } else {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Not getting DHT peers");
             }
 
 
             // we could try and total the unique peers but that's too hard for now
             snark.setTrackerSeenPeers(maxSeenPeers);
-            if (!runStarted)
-                _util.debug("         Retrying in one minute...", Snark.DEBUG);
+
+            if (stop)
+                return;
+
+            try {
+                // Sleep some minutes...
+                // Sleep the minimum interval for all the trackers, but 60s minimum
+                int delay;
+                int random = r.nextInt(120*1000);
+                if (completed && runStarted)
+                  delay = 3*SLEEP*60*1000 + random;
+                else if (snark.getTrackerProblems() != null && ++consecutiveFails < MAX_CONSEC_FAILS)
+                  delay = INITIAL_SLEEP;
+                else if ((!runStarted) && _runCount < MAX_CONSEC_FAILS)
+                  delay = INITIAL_SLEEP;
+                else
+                  // sleep a while, when we wake up we will contact only the trackers whose intervals have passed
+                  delay = SLEEP*60*1000 + random;
+
+                if (delay > 20*1000) {
+                  // put ourselves on SimpleTimer2
+                  if (_log.shouldLog(Log.DEBUG))
+                      _log.debug("Requeueing in " + DataHelper.formatDuration(delay) + ": " + Thread.currentThread().getName());
+                  queueLoop(delay);
+                  return;
+                } else if (delay > 0) {
+                  Thread.sleep(delay);
+                }
+              } catch(InterruptedException interrupt) {}
           } // *** end of while loop
       } // try
     catch (Throwable t)
       {
-        _util.debug("TrackerClient: " + t, Snark.ERROR, t);
+        _log.error("TrackerClient: " + t, t);
         if (t instanceof OutOfMemoryError)
             throw (OutOfMemoryError)t;
       }
-    finally
-      {
-        // Local DHT tracker unannounce
-        if (_util.getDHT() != null)
-            _util.getDHT().unannounce(snark.getInfoHash());
+  }
+
+  /**
+   *  Creates a thread for each tracker in parallel if tunnel is still open
+   *  @since 0.9.1
+   */
+  private void unannounce() {
+      // Local DHT tracker unannounce
+      DHT dht = _util.getDHT();
+      if (dht != null)
+          dht.unannounce(snark.getInfoHash());
+      int i = 0;
+      for (Tracker tr : trackers) {
+          if (_util.connected() &&
+              tr.started && (!tr.stop) && tr.trackerProblems == null) {
+              try {
+                  (new I2PAppThread(new Unannouncer(tr), _threadName + " U" + (++i), true)).start();
+              } catch (OutOfMemoryError oom) {
+                  // probably ran out of threads, ignore
+                  tr.reset();
+              }
+          } else {
+              tr.reset();
+          }
+      }
+  }
+
+  /**
+   *  Send "stopped" to a single tracker
+   *  @since 0.9.1
+   */
+  private class Unannouncer implements Runnable {
+     private final Tracker tr;
+
+     public Unannouncer(Tracker tr) {
+         this.tr = tr;
+     }
+
+     public void run() {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Running unannounce " + _threadName + " to " + tr.announce);
+        long uploaded = coordinator.getUploaded();
+        long downloaded = coordinator.getDownloaded();
+        long left = coordinator.getLeft();
         try
           {
-            // try to contact everybody we can
             // Don't try to restart I2CP connection just to say goodbye
-            for (Iterator iter = trackers.iterator(); iter.hasNext(); ) {
-              if (!_util.connected()) return;
-              Tracker tr = (Tracker)iter.next();
-              if (tr.started && (!tr.stop) && tr.trackerProblems == null)
-                  doRequest(tr, infoHash, peerID, uploaded,
+              if (_util.connected()) {
+                  if (tr.started && (!tr.stop) && tr.trackerProblems == null)
+                      doRequest(tr, infoHash, peerID, uploaded,
                                          downloaded, left, STOPPED_EVENT);
-            }
+              }
           }
         catch(IOException ioe) { /* ignored */ }
-      }
-    
+        tr.reset();
+     }
   }
   
   private TrackerInfo doRequest(Tracker tr, String infoHash,
@@ -443,26 +608,40 @@ public class TrackerClient extends I2PAppThread
                                 long downloaded, long left, String event)
     throws IOException
   {
-    // What do we send for left in magnet mode? Can we omit it?
-    long tleft = left >= 0 ? left : 1;
-    String s = tr.announce
-      + "?info_hash=" + infoHash
-      + "&peer_id=" + peerID
-      + "&port=" + port
-      + "&ip=" + _util.getOurIPString() + ".i2p"
-      + "&uploaded=" + uploaded
-      + "&downloaded=" + downloaded
-      + "&left=" + tleft
-      + "&compact=1"   // NOTE: opentracker will return 400 for &compact alone
-      + ((! event.equals(NO_EVENT)) ? ("&event=" + event) : "");
-    if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers())
-        s += "&numwant=0";
+    StringBuilder buf = new StringBuilder(512);
+    buf.append(tr.announce);
+    if (tr.announce.contains("?"))
+        buf.append('&');
     else
-        s += "&numwant=" + _util.getMaxConnections();
-    _util.debug("Sending TrackerClient request: " + s, Snark.INFO);
+        buf.append('?');
+    buf.append("info_hash=").append(infoHash)
+       .append("&peer_id=").append(peerID)
+       .append("&port=").append(port)
+       .append("&ip=" ).append(_util.getOurIPString()).append(".i2p")
+       .append("&uploaded=").append(uploaded)
+       .append("&downloaded=").append(downloaded)
+       .append("&left=");
+    // What do we send for left in magnet mode? Can we omit it?
+    if (left >= 0)
+        buf.append(left);
+    else
+        buf.append('1');
+    buf.append("&compact=1");  // NOTE: opentracker will return 400 for &compact alone
+    if (! event.equals(NO_EVENT))
+        buf.append("&event=").append(event);
+    buf.append("&numwant=");
+    if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needOutboundPeers())
+        buf.append('0');
+    else
+        buf.append(_util.getMaxConnections());
+    String s = buf.toString();
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Sending TrackerClient request: " + s);
       
     tr.lastRequestTime = System.currentTimeMillis();
-    File fetched = _util.get(s);
+    // Don't wait for a response to stopped when shutting down
+    boolean fast = _fastUnannounce && event.equals(STOPPED_EVENT);
+    File fetched = _util.get(s, true, fast ? -1 : 0);
     if (fetched == null) {
         throw new IOException("Error fetching " + s);
     }
@@ -472,14 +651,15 @@ public class TrackerClient extends I2PAppThread
         in = new FileInputStream(fetched);
 
         TrackerInfo info = new TrackerInfo(in, snark.getID(),
-                                           snark.getInfoHash(), snark.getMetaInfo());
-        _util.debug("TrackerClient response: " + info, Snark.INFO);
+                                           snark.getInfoHash(), snark.getMetaInfo(), _util);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("TrackerClient response: " + info);
 
         String failure = info.getFailureReason();
         if (failure != null)
           throw new IOException(failure);
 
-        tr.interval = info.getInterval() * 1000;
+        tr.interval = Math.max(MIN_TRACKER_ANNOUNCE_INTERVAL, info.getInterval() * 1000l);
         return info;
     } finally {
         if (in != null) try { in.close(); } catch (IOException ioe) {}
@@ -521,7 +701,7 @@ public class TrackerClient extends I2PAppThread
    *  @return true for i2p hosts only
    *  @since 0.7.12
    */
-  static boolean isValidAnnounce(String ann) {
+  public static boolean isValidAnnounce(String ann) {
     URL url;
     try {
        url = new URL(ann);
@@ -551,6 +731,13 @@ public class TrackerClient extends I2PAppThread
           announce = a;
           isPrimary = p;
           interval = INITIAL_SLEEP;
+      }
+
+      /**
+       *  Call before restarting
+       *  @since 0.9.1
+       */
+      public void reset() {
           lastRequestTime = 0;
           trackerProblems = null;
           stop = false;

apps/i2psnark/java/src/org/klomp/snark/TrackerInfo.java

@@ -38,7 +38,7 @@ import org.klomp.snark.bencode.InvalidBEncodingException;
  *  Compact format 1 - a list of hashes - early format for testing
  *  Compact format 2 - One big string of concatenated hashes - official format
  */
-public class TrackerInfo
+class TrackerInfo
 {
   private final String failure_reason;
   private final int interval;
@@ -47,19 +47,19 @@ public class TrackerInfo
   private int incomplete;
 
   /** @param metainfo may be null */
-  public TrackerInfo(InputStream in, byte[] my_id, byte[] infohash, MetaInfo metainfo)
+  public TrackerInfo(InputStream in, byte[] my_id, byte[] infohash, MetaInfo metainfo, I2PSnarkUtil util)
     throws IOException
   {
-    this(new BDecoder(in), my_id, infohash, metainfo);
+    this(new BDecoder(in), my_id, infohash, metainfo, util);
   }
 
-  private TrackerInfo(BDecoder be, byte[] my_id, byte[] infohash, MetaInfo metainfo)
+  private TrackerInfo(BDecoder be, byte[] my_id, byte[] infohash, MetaInfo metainfo, I2PSnarkUtil util)
     throws IOException
   {
-    this(be.bdecodeMap().getMap(), my_id, infohash, metainfo);
+    this(be.bdecodeMap().getMap(), my_id, infohash, metainfo, util);
   }
 
-  private TrackerInfo(Map m, byte[] my_id, byte[] infohash, MetaInfo metainfo)
+  private TrackerInfo(Map m, byte[] my_id, byte[] infohash, MetaInfo metainfo, I2PSnarkUtil util)
     throws IOException
   {
     BEValue reason = (BEValue)m.get("failure reason");
@@ -85,10 +85,10 @@ public class TrackerInfo
             Set<Peer> p;
             try {
               // One big string (the official compact format)
-              p = getPeers(bePeers.getBytes(), my_id, infohash, metainfo);
+              p = getPeers(bePeers.getBytes(), my_id, infohash, metainfo, util);
             } catch (InvalidBEncodingException ibe) {
               // List of Dictionaries or List of Strings
-              p = getPeers(bePeers.getList(), my_id, infohash, metainfo);
+              p = getPeers(bePeers.getList(), my_id, infohash, metainfo, util);
             }
             peers = p;
         }
@@ -124,7 +124,7 @@ public class TrackerInfo
 ******/
 
   /** List of Dictionaries or List of Strings */
-  private static Set<Peer> getPeers(List<BEValue> l, byte[] my_id, byte[] infohash, MetaInfo metainfo)
+  private static Set<Peer> getPeers(List<BEValue> l, byte[] my_id, byte[] infohash, MetaInfo metainfo, I2PSnarkUtil util)
     throws IOException
   {
     Set<Peer> peers = new HashSet(l.size());
@@ -138,7 +138,7 @@ public class TrackerInfo
             try {
                 // Case 2 - compact - A list of 32-byte binary strings (hashes)
                 // This was just for testing and is not the official format
-                peerID = new PeerID(bev.getBytes());
+                peerID = new PeerID(bev.getBytes(), util);
             } catch (InvalidBEncodingException ibe2) {
                 // don't let one bad entry spoil the whole list
                 //Snark.debug("Discarding peer from list: " + ibe, Snark.ERROR);
@@ -157,7 +157,7 @@ public class TrackerInfo
    *  One big string of concatenated 32-byte hashes
    *  @since 0.8.1
    */
-  private static Set<Peer> getPeers(byte[] l, byte[] my_id, byte[] infohash, MetaInfo metainfo)
+  private static Set<Peer> getPeers(byte[] l, byte[] my_id, byte[] infohash, MetaInfo metainfo, I2PSnarkUtil util)
     throws IOException
   {
     int count = l.length / HASH_LENGTH;
@@ -168,7 +168,7 @@ public class TrackerInfo
         byte[] hash = new byte[HASH_LENGTH];
         System.arraycopy(l, i * HASH_LENGTH, hash, 0, HASH_LENGTH);
         try {
-            peerID = new PeerID(hash);
+            peerID = new PeerID(hash, util);
         } catch (InvalidBEncodingException ibe) {
             // won't happen
             continue;
@@ -195,6 +195,7 @@ public class TrackerInfo
     return failure_reason;
   }
 
+  /** in seconds */
   public int getInterval()
   {
     return interval;

apps/i2psnark/java/src/org/klomp/snark/dht/DHT.java

@@ -4,7 +4,7 @@ package org.klomp.snark.dht;
  *  GPLv2
  */
 
-import java.util.List;
+import java.util.Collection;
 
 import net.i2p.data.Destination;
 import net.i2p.data.Hash;
@@ -12,15 +12,21 @@ import net.i2p.data.Hash;
 
 /**
  * Stub for KRPC
+ * @since 0.8.4
  */
 public interface DHT {
 
 
     /**
-     *  @return The UDP port that should be included in a PORT message.
+     *  @return The UDP query port
      */
     public int getPort();
 
+    /**
+     *  @return The UDP response port
+     */
+    public int getRPort();
+
     /**
      *  Ping. We don't have a NID yet so the node is presumed
      *  to be absent from our DHT.
@@ -37,9 +43,9 @@ public interface DHT {
      *  @param ih the Info Hash (torrent)
      *  @param max maximum number of peers to return
      *  @param maxWait the maximum time to wait (ms) must be > 0
-     *  @return list or empty list (never null)
+     *  @return possibly empty (never null)
      */
-    public List<Hash> getPeers(byte[] ih, int max, long maxWait);
+    public Collection<Hash> getPeers(byte[] ih, int max, long maxWait);
 
     /**
      *  Announce to ourselves.
@@ -79,4 +85,19 @@ public interface DHT {
      *  @return the number of successful announces, not counting ourselves.
      */
     public int announce(byte[] ih, int max, long maxWait);
+
+    /**
+     * Stop everything.
+     */
+    public void stop();
+
+    /**
+     * Known nodes, not estimated total network size.
+     */
+    public int size();
+
+    /**
+     * Debug info, HTML formatted
+     */
+    public String renderStatusHTML();
 }

apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java

@@ -0,0 +1,166 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.i2p.I2PAppContext;
+import net.i2p.crypto.SHA1Hash;
+import net.i2p.data.DataHelper;
+import net.i2p.kademlia.KBucketSet;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer2;
+
+/**
+ *  All the nodes we know about, stored as a mapping from
+ *  node ID to a Destination and Port.
+ *
+ *  And a real Kademlia routing table, which stores node IDs only.
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class DHTNodes {
+
+    private final I2PAppContext _context;
+    private long _expireTime;
+    private final Log _log;
+    private final ConcurrentHashMap<NID, NodeInfo> _nodeMap;
+    private final KBucketSet<NID> _kad;
+    private volatile boolean _isRunning;
+
+    /** stagger with other cleaners */
+    private static final long CLEAN_TIME = 187*1000;
+    /** how long since last heard from do we delete  - BEP 5 says 15 minutes */
+    private static final long MAX_EXPIRE_TIME = 30*60*1000;
+    private static final long MIN_EXPIRE_TIME = 10*60*1000;
+    private static final long DELTA_EXPIRE_TIME = 3*60*1000;
+    private static final int MAX_PEERS = 799;
+    /** Buckets older than this are refreshed - BEP 5 says 15 minutes */
+    private static final long MAX_BUCKET_AGE = 15*60*1000;
+    private static final int KAD_K = 8;
+    private static final int KAD_B = 1;
+
+    public DHTNodes(I2PAppContext ctx, NID me) {
+        _context = ctx;
+        _expireTime = MAX_EXPIRE_TIME;
+        _log = _context.logManager().getLog(DHTNodes.class);
+        _nodeMap = new ConcurrentHashMap();
+        _kad = new KBucketSet(ctx, me, KAD_K, KAD_B, new KBTrimmer(ctx, KAD_K));
+    }
+
+    public void start() {
+        _isRunning = true;
+        new Cleaner();
+    }
+
+    public void stop() {
+        clear();
+        _isRunning = false;
+    }
+
+    // begin ConcurrentHashMap methods
+
+    public int size() {
+        return _nodeMap.size();
+    }
+
+    public void clear() {
+        _kad.clear();
+        _nodeMap.clear();
+    }
+
+    public NodeInfo get(NID nid) {
+        return _nodeMap.get(nid);
+    }
+
+    /**
+     *  @return the old value if present, else null
+     */
+    public NodeInfo putIfAbsent(NodeInfo nInfo) {
+        _kad.add(nInfo.getNID());
+        return _nodeMap.putIfAbsent(nInfo.getNID(), nInfo);
+    }
+
+    public NodeInfo remove(NID nid) {
+        _kad.remove(nid);
+        return _nodeMap.remove(nid);
+    }
+
+    public Collection<NodeInfo> values() {
+        return _nodeMap.values();
+    }
+
+    // end ConcurrentHashMap methods
+
+    /**
+     *  DHT
+     *  @param h either a InfoHash or a NID
+     */
+    public List<NodeInfo> findClosest(SHA1Hash h, int numWant) {
+        NID key;
+        if (h instanceof NID)
+            key = (NID) h;
+        else
+            key = new NID(h.getData());
+        List<NID> keys = _kad.getClosest(key, numWant);
+        List<NodeInfo> rv = new ArrayList(keys.size());
+        for (NID nid : keys) {
+            NodeInfo ninfo = _nodeMap.get(nid);
+            if (ninfo != null)
+                rv.add(ninfo);
+        }
+        return rv;
+    }
+
+    /**
+     *  DHT - get random keys to explore
+     */
+    public List<NID> getExploreKeys() {
+        return _kad.getExploreKeys(MAX_BUCKET_AGE);
+    }
+
+    /** */
+    private class Cleaner extends SimpleTimer2.TimedEvent {
+
+        public Cleaner() {
+            super(SimpleTimer2.getInstance(), CLEAN_TIME);
+        }
+
+        public void timeReached() {
+            if (!_isRunning)
+                return;
+            long now = _context.clock().now();
+            int peerCount = 0;
+            for (Iterator<NodeInfo> iter = DHTNodes.this.values().iterator(); iter.hasNext(); ) {
+                 NodeInfo peer = iter.next();
+                 if (peer.lastSeen() < now - _expireTime) {
+                     iter.remove();
+                     _kad.remove(peer.getNID());
+                 } else {
+                     peerCount++;
+                }
+            }
+
+            if (peerCount > MAX_PEERS)
+                _expireTime = Math.max(_expireTime - DELTA_EXPIRE_TIME, MIN_EXPIRE_TIME);
+            else
+                _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME);
+
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("DHT storage cleaner done, now with " +
+                         peerCount + " peers, " +
+                         DataHelper.formatDuration(_expireTime) + " expiration");
+
+            schedule(CLEAN_TIME);
+        }
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java

@@ -0,0 +1,168 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, relicensed to GPLv2
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer2;
+
+/**
+ * The tracker stores peers, i.e. Dest hashes (not nodes).
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class DHTTracker {
+
+    private final I2PAppContext _context;
+    private final Torrents _torrents;
+    private long _expireTime;
+    private final Log _log;
+    private volatile boolean _isRunning;
+    /** not current, updated by cleaner */
+    private int _peerCount;
+    /** not current, updated by cleaner */
+    private int _torrentCount;
+
+    /** stagger with other cleaners */
+    private static final long CLEAN_TIME = 199*1000;
+    private static final long MAX_EXPIRE_TIME = 45*60*1000;
+    private static final long MIN_EXPIRE_TIME = 15*60*1000;
+    private static final long DELTA_EXPIRE_TIME = 3*60*1000;
+    private static final int MAX_PEERS = 2000;
+    private static final int MAX_PEERS_PER_TORRENT = 150;
+
+    DHTTracker(I2PAppContext ctx) {
+        _context = ctx;
+        _torrents = new Torrents();
+        _expireTime = MAX_EXPIRE_TIME;
+        _log = _context.logManager().getLog(DHTTracker.class);
+    }
+
+    public void start() {
+        _isRunning = true;
+        new Cleaner();
+    }
+
+    void stop() {
+        _torrents.clear();
+        _isRunning = false;
+    }
+
+    void announce(InfoHash ih, Hash hash) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Announce " + hash + " for " + ih);
+        Peers peers = _torrents.get(ih);
+        if (peers == null) {
+            peers = new Peers();
+            Peers peers2 = _torrents.putIfAbsent(ih, peers);
+            if (peers2 != null)
+                peers = peers2;
+        }
+
+        Peer peer = new Peer(hash.getData());
+        Peer peer2 = peers.putIfAbsent(peer, peer);
+        if (peer2 != null)
+            peer = peer2;
+        peer.setLastSeen(_context.clock().now());
+    }
+
+    void unannounce(InfoHash ih, Hash hash) {
+        Peers peers = _torrents.get(ih);
+        if (peers == null)
+            return;
+        Peer peer = new Peer(hash.getData());
+        peers.remove(peer);
+    }
+
+    /**
+     *  Caller's responsibility to remove himself from the list
+     *  @return list or empty list (never null)
+     */
+    List<Hash> getPeers(InfoHash ih, int max) {
+        Peers peers = _torrents.get(ih);
+        if (peers == null)
+            return Collections.EMPTY_LIST;
+
+        int size = peers.size();
+        List<Hash> rv = new ArrayList(peers.values());
+        if (max < size) {
+                Collections.shuffle(rv, _context.random());
+                rv = rv.subList(0, max);
+        }
+        return rv;
+    }
+
+    /**
+     * Debug info, HTML formatted
+     */
+    public void renderStatusHTML(StringBuilder buf) {
+        buf.append("DHT tracker: ").append(_torrentCount).append(" torrents ")
+           .append(_peerCount).append(" peers ")
+           .append(DataHelper.formatDuration(_expireTime)).append(" expiration<br>");
+    }
+
+    private class Cleaner extends SimpleTimer2.TimedEvent {
+
+        public Cleaner() {
+            super(SimpleTimer2.getInstance(), CLEAN_TIME);
+        }
+
+        public void timeReached() {
+            if (!_isRunning)
+                return;
+            long now = _context.clock().now();
+            int torrentCount = 0;
+            int peerCount = 0;
+            for (Iterator<Peers> iter = _torrents.values().iterator(); iter.hasNext(); ) {
+                Peers p = iter.next();
+                int recent = 0;
+                for (Iterator<Peer> iterp = p.values().iterator(); iterp.hasNext(); ) {
+                     Peer peer = iterp.next();
+                     if (peer.lastSeen() < now - _expireTime)
+                         iterp.remove();
+                     else {
+                         recent++;
+                         peerCount++;
+                     }
+                }
+                if (recent > MAX_PEERS_PER_TORRENT) {
+                    // too many, delete at random
+                    // TODO per-torrent adjustable expiration?
+                    for (Iterator<Peer> iterp = p.values().iterator(); iterp.hasNext() && p.size() > MAX_PEERS_PER_TORRENT; ) {
+                         iterp.next();
+                         iterp.remove();
+                         peerCount--;
+                    }
+                    torrentCount++;
+                } else if (recent <= 0) {
+                    iter.remove();
+                } else {
+                    torrentCount++;
+                }
+            }
+
+            if (peerCount > MAX_PEERS)
+                _expireTime = Math.max(_expireTime - DELTA_EXPIRE_TIME, MIN_EXPIRE_TIME);
+            else
+                _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME);
+
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("DHT tracker cleaner done, now with " +
+                         torrentCount + " torrents, " +
+                         peerCount + " peers, " +
+                         DataHelper.formatDuration(_expireTime) + " expiration");
+            _peerCount = peerCount;
+            _torrentCount = torrentCount;
+            schedule(CLEAN_TIME);
+        }
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/InfoHash.java

@@ -0,0 +1,29 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import net.i2p.crypto.SHA1Hash;
+import org.klomp.snark.I2PSnarkUtil;
+
+/**
+ *  A 20-byte SHA1 info hash
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class InfoHash extends SHA1Hash {
+
+    public InfoHash(byte[] data) {
+        super(data);
+    }
+
+    @Override
+    public String toString() {
+        if (_data == null) {
+            return super.toString();
+        } else {
+            return "[InfoHash: " + I2PSnarkUtil.toHex(_data) + ']';
+        }
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/KBTrimmer.java

@@ -0,0 +1,38 @@
+package org.klomp.snark.dht;
+
+import java.util.Set;
+
+import net.i2p.I2PAppContext;
+import net.i2p.kademlia.KBucket;
+import net.i2p.kademlia.KBucketTrimmer;
+
+/**
+ *  Removes an element older than 15 minutes, but only if the bucket hasn't changed in 5 minutes.
+ *  @since 0.9.2
+ */
+class KBTrimmer implements KBucketTrimmer<NID> {
+    private final I2PAppContext _ctx;
+    private final int _max;
+
+    private static final long MIN_BUCKET_AGE = 5*60*1000;
+    private static final long MAX_NODE_AGE = 15*60*1000;
+
+    public KBTrimmer(I2PAppContext ctx, int max) {
+        _ctx = ctx;
+        _max = max;
+    }
+
+    public boolean trim(KBucket<NID> kbucket, NID toAdd) {
+        long now = _ctx.clock().now();
+        if (kbucket.getLastChanged() > now - MIN_BUCKET_AGE)
+            return false;
+        Set<NID> entries = kbucket.getEntries();
+        for (NID nid : entries) {
+            if (nid.lastSeen() < now - MAX_NODE_AGE) {
+                if (kbucket.remove(nid))
+                    return true;
+            }
+        }
+        return entries.size() < _max;
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/MsgID.java

@@ -0,0 +1,37 @@
+package org.klomp.snark.dht;
+/*
+ *  GPLv2
+ */
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+
+/**
+ *  Used for both incoming and outgoing message IDs
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class MsgID extends ByteArray {
+
+    /** BEP 5: 2 bytes, incremented */
+    private static final int MY_TOK_LEN = 8;
+    private static final int MAX_TOK_LEN = 16;
+
+    /** outgoing - generate a random ID */
+    public MsgID(I2PAppContext ctx) {
+        super(null);
+        byte[] data = new byte[MY_TOK_LEN];
+        ctx.random().nextBytes(data);
+        setData(data);
+        setValid(MY_TOK_LEN);
+    }
+
+    /** incoming  - save the ID (arbitrary length) */
+    public MsgID(byte[] data) {
+        super(data);
+        // lets not get carried away
+        if (data.length > MAX_TOK_LEN)
+            throw new IllegalArgumentException();
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/NID.java

@@ -0,0 +1,46 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import net.i2p.crypto.SHA1Hash;
+import net.i2p.util.Clock;
+
+/**
+ *  A 20-byte peer ID, used as a Map key in lots of places.
+ *  Must be public for constructor in KBucketSet.generateRandomKey()
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+public class NID extends SHA1Hash {
+
+    private long lastSeen;
+    private int fails;
+
+    private static final int MAX_FAILS = 3;
+
+    public NID() {
+        super(null);
+    }
+
+    public NID(byte[] data) {
+        super(data);
+    }
+
+    public long lastSeen() {
+        return lastSeen;
+    }
+
+    public void setLastSeen() {
+        lastSeen = Clock.getInstance().now();
+        fails = 0;
+    }
+
+    /**
+     *  @return if more than max timeouts
+     */
+    public boolean timeout() {
+        return fails++ > MAX_FAILS;
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfo.java

@@ -0,0 +1,253 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import net.i2p.data.Base64;
+import net.i2p.data.DataFormatException;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Destination;
+import net.i2p.data.Hash;
+import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.RandomSource;
+
+/*
+ *  A Node ID, Hash, and port, and an optional Destination.
+ *  This is what DHTNodes remembers. The DHT tracker just stores Hashes.
+ *  getData() returns the 54 byte compact info (NID, Hash, port).
+ *
+ *  Things are a little tricky in KRPC since we exchange Hashes and don't
+ *  always have the Destination.
+ *  The conpact info is immutable. The Destination may be added later.
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+
+class NodeInfo extends SimpleDataStructure {
+
+    private final NID nID;
+    private final Hash hash;
+    private Destination dest;
+    private final int port;
+
+    public static final int LENGTH = NID.HASH_LENGTH + Hash.HASH_LENGTH + 2;
+
+    /**
+     * With a fake NID used for pings
+     */
+    public NodeInfo(Destination dest, int port) {
+        super();
+        this.nID = KRPC.FAKE_NID;
+        this.dest = dest;
+        this.hash = dest.calculateHash();
+        this.port = port;
+        initialize();
+    }
+
+    /**
+     * Use this if we have the full destination
+     * @throws IllegalArgumentException
+     */
+    public NodeInfo(NID nID, Destination dest, int port) {
+        super();
+        this.nID = nID;
+        this.dest = dest;
+        this.hash = dest.calculateHash();
+        this.port = port;
+        initialize();
+        verify();
+    }
+
+    /**
+     * No Destination yet available
+     * @throws IllegalArgumentException
+     */
+    public NodeInfo(NID nID, Hash hash, int port) {
+        super();
+        this.nID = nID;
+        this.hash = hash;
+        this.port = port;
+        initialize();
+        verify();
+    }
+
+    /**
+     * No Destination yet available
+     * @param compactInfo 20 byte node ID, 32 byte destHash, 2 byte port
+     * @param offset starting at this offset in compactInfo
+     * @throws IllegalArgumentException
+     * @throws AIOOBE
+     */
+    public NodeInfo(byte[] compactInfo, int offset) {
+        super();
+        byte[] d = new byte[LENGTH];
+        System.arraycopy(compactInfo, offset, d, 0, LENGTH);
+        setData(d);
+        byte[] ndata = new byte[NID.HASH_LENGTH];
+        System.arraycopy(d, 0, ndata, 0, NID.HASH_LENGTH);
+        this.nID = new NID(ndata);
+        this.hash = Hash.create(d, NID.HASH_LENGTH);
+        this.port = (int) DataHelper.fromLong(d, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2);
+        if (port <= 0 || port >= 65535)
+            throw new IllegalArgumentException("Bad port");
+        verify();
+    }
+
+    /**
+     * Create from persistent storage string.
+     * Format: NID:Hash:Destination:port
+     * First 3 in base 64; Destination may be empty string
+     * @throws IllegalArgumentException
+     */
+    public NodeInfo(String s) throws DataFormatException {
+        super();
+        String[] parts = s.split(":", 4);
+        if (parts.length != 4)
+            throw new DataFormatException("Bad format");
+        byte[] nid = Base64.decode(parts[0]);
+        if (nid == null)
+            throw new DataFormatException("Bad NID");
+        nID = new NID(nid);
+        byte[] h = Base64.decode(parts[1]);
+        if (h == null)
+            throw new DataFormatException("Bad hash");
+        //hash = new Hash(h);
+        hash = Hash.create(h);
+        if (parts[2].length() > 0)
+            dest = new Destination(parts[2]);
+        try {
+            port = Integer.parseInt(parts[3]);
+        } catch (NumberFormatException nfe) {
+            throw new DataFormatException("Bad port", nfe);
+        }
+        initialize();
+    }
+
+
+    /**
+     * Creates 54-byte compact info
+     * @throws IllegalArgumentException
+     */
+    private void initialize() {
+        if (port <= 0 || port >= 65535)
+            throw new IllegalArgumentException("Bad port");
+        byte[] compactInfo = new byte[LENGTH];
+        System.arraycopy(nID.getData(), 0, compactInfo, 0, NID.HASH_LENGTH);
+        System.arraycopy(hash.getData(), 0, compactInfo, NID.HASH_LENGTH, Hash.HASH_LENGTH);
+        DataHelper.toLong(compactInfo, NID.HASH_LENGTH + Hash.HASH_LENGTH, 2, port);
+        setData(compactInfo);
+    }
+
+    /**
+     * Generate a secure NID that matches the Hash and port.
+     * Rules: First 4 bytes must match Hash.
+     * Next 2 bytes must match Hash ^ port.
+     * Remaining bytes may be random.
+     *
+     * @throws IllegalArgumentException
+     */
+    public static NID generateNID(Hash h, int p, RandomSource random) {
+        byte[] n = new byte[NID.HASH_LENGTH];
+        System.arraycopy(h.getData(), 0, n, 0, 6);
+        n[4] ^= (byte) (p >> 8);
+        n[5] ^= (byte) p;
+        random.nextBytes(n, 6, NID.HASH_LENGTH - 6);
+        return new NID(n);
+    }
+
+    /**
+     * Verify the NID matches the Hash.
+     * See generateNID() for requirements.
+     * @throws IllegalArgumentException on mismatch
+     */
+    private void verify() {
+        if (!KRPC.SECURE_NID)
+            return;
+        byte[] nb = nID.getData();
+        byte[] hb = hash.getData();
+        if ((!DataHelper.eq(nb, 0, hb, 0, 4)) ||
+            ((nb[4] ^ (port >> 8)) & 0xff) != (hb[4] & 0xff) ||
+            ((nb[5] ^ port) & 0xff) != (hb[5] & 0xff))
+            throw new IllegalArgumentException("NID/Hash mismatch");
+    }
+
+    public int length() {
+        return LENGTH;
+    }
+
+    public NID getNID() {
+        return this.nID;
+    }
+
+    /** @return may be null if we don't have it */
+    public Destination getDestination() {
+        return this.dest;
+    }
+
+    public Hash getHash() {
+        return this.hash;
+    }
+
+    @Override
+    public Hash calculateHash() {
+        return this.hash;
+    }
+
+    /**
+     * This can come in later but the hash must match.
+     * @throws IllegalArgumentException if hash of dest doesn't match previous hash
+     */
+    public void setDestination(Destination dest) throws IllegalArgumentException {
+        if (this.dest != null)
+            return;
+        if (!dest.calculateHash().equals(this.hash))
+            throw new IllegalArgumentException("Hash mismatch, was: " + this.hash + " new: " + dest.calculateHash());
+        this.dest = dest;
+    }
+
+    public int getPort() {
+        return this.port;
+    }
+
+    public long lastSeen() {
+        return nID.lastSeen();
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() ^ nID.hashCode() ^ port;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        try {
+            NodeInfo ni = (NodeInfo) o;
+            // assume dest matches, ignore it
+            return this.hash.equals(ni.hash) && nID.equals(ni.nID) && port == ni.port;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "NodeInfo: " + nID + ' ' + hash + " port: " + port + (dest != null ? " known dest" : " null dest");
+    }
+
+    /**
+     * To persistent storage string.
+     * Format: NID:Hash:Destination:port
+     * First 3 in base 64; Destination may be empty string
+     */
+    public String toPersistentString() {
+        StringBuilder buf = new StringBuilder(650);
+        buf.append(nID.toBase64()).append(':');
+        buf.append(hash.toBase64()).append(':');
+        if (dest != null)
+            buf.append(dest.toBase64());
+        buf.append(':').append(port);
+        return buf.toString();
+    }
+
+}

apps/i2psnark/java/src/org/klomp/snark/dht/NodeInfoComparator.java

@@ -0,0 +1,31 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import java.util.Comparator;
+
+import net.i2p.crypto.SHA1Hash;
+import net.i2p.data.DataHelper;
+
+/**
+ *  Closest to a InfoHash or NID key.
+ *  Use for NodeInfos.
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class NodeInfoComparator implements Comparator<NodeInfo> {
+    private final byte[] _base;
+
+    public NodeInfoComparator(SHA1Hash h) {
+        _base = h.getData();
+    }
+
+    public int compare(NodeInfo lhs, NodeInfo rhs) {
+        byte lhsDelta[] = DataHelper.xor(lhs.getNID().getData(), _base);
+        byte rhsDelta[] = DataHelper.xor(rhs.getNID().getData(), _base);
+        return DataHelper.compareTo(lhsDelta, rhsDelta);
+    }
+
+}

apps/i2psnark/java/src/org/klomp/snark/dht/Peer.java

@@ -0,0 +1,30 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import net.i2p.data.Hash;
+
+/**
+ *  A single peer for a single torrent.
+ *  This is what the DHT tracker remembers.
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class Peer extends Hash {
+
+    private long lastSeen;
+
+    public Peer(byte[] data) {
+        super(data);
+    }
+
+    public long lastSeen() {
+        return lastSeen;
+    }
+
+    public void setLastSeen(long now) {
+        lastSeen = now;
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/Peers.java

@@ -0,0 +1,21 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, modded and relicensed to GPLv2
+ */
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.i2p.data.Hash;
+
+/**
+ *  All the peers for a single torrent
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class Peers extends ConcurrentHashMap<Hash, Peer> {
+
+    public Peers() {
+        super(8);
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java

@@ -0,0 +1,85 @@
+package org.klomp.snark.dht;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.DataFormatException;
+import net.i2p.util.Log;
+import net.i2p.util.SecureFileOutputStream;
+
+/**
+ *  Retrieve / Store the local DHT in a file
+ *
+ *  @since 0.9.2
+ */
+abstract class PersistDHT {
+
+    private static final long MAX_AGE = 60*60*1000;
+
+    public static synchronized void loadDHT(KRPC krpc, File file) {
+        Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class);
+        int count = 0;
+        FileInputStream in = null;
+        try {
+            in = new FileInputStream(file);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in, "ISO-8859-1"));
+            String line = null;
+            while ( (line = br.readLine()) != null) {
+                if (line.startsWith("#"))
+                    continue;
+                try {
+                    krpc.heardAbout(new NodeInfo(line));
+                    count++;
+                    // TODO limit number? this will flush the router's SDS caches
+                } catch (IllegalArgumentException iae) {
+                    if (log.shouldLog(Log.WARN))
+                        log.warn("Error reading DHT entry", iae);
+                } catch (DataFormatException dfe) {
+                    if (log.shouldLog(Log.WARN))
+                        log.warn("Error reading DHT entry", dfe);
+                }
+            }
+        } catch (IOException ioe) {
+            if (log.shouldLog(Log.WARN) && file.exists())
+                log.warn("Error reading the DHT File", ioe);
+        } finally {
+            if (in != null) try { in.close(); } catch (IOException ioe) {}
+        }
+        if (log.shouldLog(Log.INFO))
+            log.info("Loaded " + count + " nodes from " + file);
+    }
+
+    public static synchronized void saveDHT(DHTNodes nodes, File file) {
+        if (nodes.size() <= 0)
+            return;
+        Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class);
+        int count = 0;
+        long maxAge = I2PAppContext.getGlobalContext().clock().now() - MAX_AGE;
+        PrintWriter out = null;
+        try {
+            out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(file), "ISO-8859-1")));
+            out.println("# DHT nodes, format is NID:Hash:Destination:port");
+            for (NodeInfo ni : nodes.values()) {
+                 if (ni.lastSeen() < maxAge)
+                     continue;
+                 // DHTNodes shouldn't contain us, if that changes check here
+                 out.println(ni.toPersistentString());
+                 count++;
+            }
+        } catch (IOException ioe) {
+            if (log.shouldLog(Log.WARN))
+                log.warn("Error writing the DHT File", ioe);
+        } finally {
+            if (out != null) out.close();
+        }
+        if (log.shouldLog(Log.INFO))
+            log.info("Stored " + count + " nodes to " + file);
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/Token.java

@@ -0,0 +1,75 @@
+package org.klomp.snark.dht;
+/*
+ *  GPLv2
+ */
+
+import java.util.Date;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+
+/**
+ *  Used for Both outgoing and incoming tokens
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class Token extends ByteArray {
+
+    private static final int MY_TOK_LEN = 8;
+    private static final int MAX_TOK_LEN = 64;
+    private final long lastSeen;
+
+    /** outgoing - generate a random token */
+    public Token(I2PAppContext ctx) {
+        super(null);
+        byte[] data = new byte[MY_TOK_LEN];
+        ctx.random().nextBytes(data);
+        setData(data);
+        setValid(MY_TOK_LEN);
+        lastSeen = ctx.clock().now();
+    }
+
+    /** incoming  - save the token (arbitrary length) */
+    public Token(I2PAppContext ctx, byte[] data) {
+        super(data);
+        // lets not get carried away
+        if (data.length > MAX_TOK_LEN)
+            throw new IllegalArgumentException();
+        lastSeen = ctx.clock().now();
+    }
+
+    /** incoming  - for lookup only, not storage, lastSeen is 0 */
+    public Token(byte[] data) {
+        super(data);
+        lastSeen = 0;
+    }
+
+    public long lastSeen() {
+        return lastSeen;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(64);
+        buf.append("[Token: ");
+        byte[] bs = getData();
+        if (bs.length == 0) {
+            buf.append("0 bytes");
+        } else {
+            buf.append(bs.length).append(" bytes: 0x");
+            // backwards, but the same way BEValue does it
+            for (int i = 0; i < bs.length; i++) {
+                int b = bs[i] & 0xff;
+                if (b < 16)
+                    buf.append('0');
+                buf.append(Integer.toHexString(b));
+            }
+        }
+        if (lastSeen > 0)
+            buf.append(" created ").append((new Date(lastSeen)).toString());
+        buf.append(']');
+        return buf.toString();
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/TokenKey.java

@@ -0,0 +1,20 @@
+package org.klomp.snark.dht;
+/*
+ *  GPLv2
+ */
+
+import net.i2p.crypto.SHA1Hash;
+import net.i2p.data.DataHelper;
+
+/**
+ *  Used to index incoming Tokens
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class TokenKey extends SHA1Hash {
+
+    public TokenKey(NID nID, InfoHash ih) {
+        super(DataHelper.xor(nID.getData(), ih.getData()));
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/Torrents.java

@@ -0,0 +1,19 @@
+package org.klomp.snark.dht;
+/*
+ *  From zzzot, relicensed to GPLv2
+ */
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ *  All the torrents
+ *
+ * @since 0.9.2
+ * @author zzz
+ */
+class Torrents extends ConcurrentHashMap<InfoHash, Peers> {
+
+    public Torrents() {
+        super();
+    }
+}