* 2005-10-14 0.6.1.3 released

2005-10-14  jrandom
    * Added a key explaining peers.jsp a bit (thanks tethra!)

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnel.java

@@ -291,6 +291,7 @@ public class I2PTunnel implements Logging, EventDispatcher {
         l.log("genkeys <privkeyfile> [<pubkeyfile>]");
         l.log("gentextkeys");
         l.log("client <port> <pubkey>[,<pubkey,...]|file:<pubkeyfile> [<sharedClient>]");
+        l.log("ircclient <port> <pubkey>[,<pubkey,...]|file:<pubkeyfile> [<sharedClient>]");
         l.log("httpclient <port> [<sharedClient>] [<proxy>]");
         l.log("lookup <name>");
         l.log("quit");
@@ -596,6 +597,60 @@ public class I2PTunnel implements Logging, EventDispatcher {
         }
     }
 
+    /**
+     * Run an IRC client on the given port number 
+     *
+     * Sets the event "ircclientTaskId" = Integer(taskId) after the tunnel has been started (or -1 on error).
+     * Also sets "ircclientStatus" = "ok" or "error" after the client tunnel has started.
+     * parameter sharedClient is a String, either "true" or "false"
+     *
+     * @param args {portNumber,destinationBase64 or "file:filename" [, sharedClient]}
+     * @param l logger to receive events and output
+     */
+    public void runIrcClient(String args[], Logging l) {
+        if (args.length >= 2 && args.length <= 3) {
+            int port = -1;
+            try {
+                port = Integer.parseInt(args[0]);
+            } catch (NumberFormatException nfe) {
+                l.log("invalid port");
+                _log.error(getPrefix() + "Port specified is not valid: " + args[0], nfe);
+                notifyEvent("ircclientTaskId", new Integer(-1));
+                return;
+            }
+            
+            boolean isShared = true;
+            if (args.length > 1) {
+                if ("true".equalsIgnoreCase(args[2].trim())) {
+                    isShared = true;
+                } else if ("false".equalsIgnoreCase(args[1].trim())) {
+                    _log.warn("args[1] == [" + args[1] + "] and rejected explicitly");
+                    isShared = false;
+                } else {
+                    // isShared not specified, default to true
+                    isShared = true;
+                }
+            }
+
+            I2PTunnelTask task;
+            ownDest = !isShared;
+            try {
+                task = new I2PTunnelIRCClient(port, args[1],l, ownDest, (EventDispatcher) this, this);
+                addtask(task);
+                notifyEvent("ircclientTaskId", new Integer(task.getId()));
+            } catch (IllegalArgumentException iae) {
+                _log.error(getPrefix() + "Invalid I2PTunnel config to create an ircclient [" + host + ":"+ port + "]", iae);
+                l.log("Invalid I2PTunnel configuration [" + host + ":" + port + "]");
+                notifyEvent("ircclientTaskId", new Integer(-1));
+            }
+        } else {
+            l.log("ircclient <port> [<sharedClient>]");
+            l.log("  creates a client that filter IRC protocol.");
+            l.log("  <sharedClient> (optional) indicates if this client shares tunnels with other clients (true of false)");
+            notifyEvent("ircclientTaskId", new Integer(-1));
+        }
+    }
+    
     /**
      * Run an SOCKS tunnel on the given port number 
      *

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java

@@ -17,6 +17,7 @@ import java.util.List;
 import java.util.Properties;
 
 import net.i2p.I2PException;
+import net.i2p.I2PAppContext;
 import net.i2p.client.I2PSession;
 import net.i2p.client.streaming.I2PSocket;
 import net.i2p.client.streaming.I2PSocketManager;
@@ -31,6 +32,7 @@ import net.i2p.util.Log;
 public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runnable {
 
     private static final Log _log = new Log(I2PTunnelClientBase.class);
+    protected I2PAppContext _context;
     protected Logging l;
 
     static final long DEFAULT_CONNECT_TIMEOUT = 60 * 1000;
@@ -101,6 +103,12 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
         this.l = l;
         this.handlerName = handlerName + _clientId;
 
+        _context = tunnel.getContext();
+        _context.statManager().createRateStat("i2ptunnel.client.closeBacklog", "How many pending sockets remain when we close one due to backlog?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
+        _context.statManager().createRateStat("i2ptunnel.client.closeNoBacklog", "How many pending sockets remain when it was removed prior to backlog timeout?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
+        _context.statManager().createRateStat("i2ptunnel.client.manageTime", "How long it takes to accept a socket and fire it into an i2ptunnel runner (or queue it for the pool)?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
+        _context.statManager().createRateStat("i2ptunnel.client.buildRunTime", "How long it takes to run a queued socket into an i2ptunnel runner?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
+
         // no need to load the netDb with leaseSets for destinations that will never 
         // be looked up
         tunnel.getClientOptions().setProperty("i2cp.dontPublishLeaseSet", "true");
@@ -362,7 +370,10 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
 
             while (true) {
                 Socket s = ss.accept();
+                long before = System.currentTimeMillis();
                 manageConnection(s);
+                long total = System.currentTimeMillis() - before;
+                _context.statManager().addRateData("i2ptunnel.client.manageTime", total, total);
             }
         } catch (IOException ex) {
             _log.error("Error listening for connections on " + localPort, ex);
@@ -422,14 +433,20 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
         private Socket _s;
         public CloseEvent(Socket s) { _s = s; }
         public void timeReached() {
+            int remaining = 0;
             boolean stillWaiting = false;
             synchronized (_waitingSockets) {
                 stillWaiting = _waitingSockets.remove(_s);
+                remaining = _waitingSockets.size();
             }
             if (stillWaiting) {
                 try { _s.close(); } catch (IOException ioe) {}
-                if (_log.shouldLog(Log.INFO))
+                if (_log.shouldLog(Log.INFO)) {
+                    _context.statManager().addRateData("i2ptunnel.client.closeBacklog", remaining, 0);
                     _log.info("Closed a waiting socket because of backlog");
+                }
+            } else {
+                _context.statManager().addRateData("i2ptunnel.client.closeNoBacklog", remaining, 0);
             }
         }
     }
@@ -496,8 +513,12 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
                     }
                 } catch (InterruptedException ie) {}
                 
-                if (s != null)
+                if (s != null) {
+                    long before = System.currentTimeMillis();
                     clientConnectionRun(s);
+                    long total = System.currentTimeMillis() - before;
+                    _context.statManager().addRateData("i2ptunnel.client.buildRunTime", total, 0);
+                }
                 s = null;
             }
         }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java

@@ -502,10 +502,10 @@ public class I2PTunnelHTTPClient extends I2PTunnelClientBase implements Runnable
             String remoteID;
             
             Properties opts = new Properties();
-            opts.setProperty("i2p.streaming.inactivityTimeout", ""+120*1000);
+            //opts.setProperty("i2p.streaming.inactivityTimeout", ""+120*1000);
             // 1 == disconnect.  see ConnectionOptions in the new streaming lib, which i
             // dont want to hard link to here
-            opts.setProperty("i2p.streaming.inactivityTimeoutAction", ""+1);
+            //opts.setProperty("i2p.streaming.inactivityTimeoutAction", ""+1);
             I2PSocket i2ps = createI2PSocket(dest, getDefaultOptions(opts));
             byte[] data = newRequest.toString().getBytes("ISO-8859-1");
             Runnable onTimeout = new OnTimeout(s, s.getOutputStream(), targetRequest, usingWWWProxy, currentProxy, requestId);

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java

@@ -0,0 +1,389 @@
+package net.i2p.i2ptunnel;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import net.i2p.I2PAppContext;
+import net.i2p.client.streaming.I2PSocket;
+import net.i2p.data.DataFormatException;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Destination;
+import net.i2p.util.EventDispatcher;
+import net.i2p.util.Log;
+
+public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable {
+
+    private static final Log _log = new Log(I2PTunnelIRCClient.class);
+    
+    /** used to assign unique IDs to the threads / clients.  no logic or functionality */
+    private static volatile long __clientId = 0;
+    
+    /** list of Destination objects that we point at */
+    protected List dests;
+    private static final long DEFAULT_READ_TIMEOUT = 5*60*1000; // -1
+    protected long readTimeout = DEFAULT_READ_TIMEOUT;
+
+    /**
+     * @throws IllegalArgumentException if the I2PTunnel does not contain
+     *                                  valid config to contact the router
+     */
+    public I2PTunnelIRCClient(
+                              int localPort,
+                              String destinations,
+                              Logging l, 
+                              boolean ownDest,
+                              EventDispatcher notifyThis,
+                              I2PTunnel tunnel) throws IllegalArgumentException {
+        super(localPort, 
+              ownDest, 
+              l, 
+              notifyThis, 
+              "IRCHandler " + (++__clientId), tunnel);
+
+        StringTokenizer tok = new StringTokenizer(destinations, ",");
+        dests = new ArrayList(1);
+        while (tok.hasMoreTokens()) {
+            String destination = tok.nextToken();
+            try {
+                Destination dest = I2PTunnel.destFromName(destination);
+                if (dest == null)
+                    l.log("Could not resolve " + destination);
+                else
+                    dests.add(dest);
+            } catch (DataFormatException dfe) {
+                l.log("Bad format parsing \"" + destination + "\"");
+            }
+        }
+
+        if (dests.size() <= 0) {
+            l.log("No target destinations found");
+            notifyEvent("openClientResult", "error");
+            return;
+        }
+        
+        setName(getLocalPort() + " -> IRCClient");
+
+        startRunning();
+
+        notifyEvent("openIRCClientResult", "ok");
+    }
+    
+    protected void clientConnectionRun(Socket s) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("got a connection.");
+        Destination dest = pickDestination();
+        I2PSocket i2ps = null;
+        try {
+            i2ps = createI2PSocket(dest);
+            i2ps.setReadTimeout(readTimeout);
+            Thread in = new Thread(new IrcInboundFilter(s,i2ps));
+            in.start();
+            Thread out = new Thread(new IrcOutboundFilter(s,i2ps));
+            out.start();
+        } catch (Exception ex) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("Error connecting", ex);
+            l.log(ex.getMessage());
+            closeSocket(s);
+            if (i2ps != null) {
+                synchronized (sockLock) {
+                    mySockets.remove(sockLock);
+                }
+            }
+        }
+
+    }
+    
+    private final Destination pickDestination() {
+        int size = dests.size();
+        if (size <= 0) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("No client targets?!");
+            return null;
+        }
+        if (size == 1) // skip the rand in the most common case
+            return (Destination)dests.get(0);
+        int index = I2PAppContext.getGlobalContext().random().nextInt(size);
+        return (Destination)dests.get(index);
+    }
+
+    /*************************************************************************
+     *
+     */
+    private class IrcInboundFilter implements Runnable {
+        
+        private Socket local;
+        private I2PSocket remote;
+                
+        IrcInboundFilter(Socket _local, I2PSocket _remote) {
+            local=_local;
+            remote=_remote;
+        }
+
+        public void run() {
+            InputStream input;
+            OutputStream output;
+            try {
+                input=remote.getInputStream();
+                output=local.getOutputStream();
+            } catch (IOException e) {
+                if (_log.shouldLog(Log.ERROR))
+                    _log.error("IrcInboundFilter: no streams",e);
+                return;
+            }
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("IrcInboundFilter: Running.");
+            while(true)
+            {
+                try {
+                    String inmsg = DataHelper.readLine(input);
+                    if(inmsg==null)
+                        break;
+                    if(inmsg.endsWith("\r"))
+                        inmsg=inmsg.substring(0,inmsg.length()-1);
+                    String outmsg = inboundFilter(inmsg);
+                    if(outmsg!=null)
+                    {
+                        if(!inmsg.equals(outmsg)) {
+                            if (_log.shouldLog(Log.WARN)) {
+                                _log.warn("inbound FILTERED: "+outmsg);
+                                _log.warn(" - inbound was: "+inmsg);
+                            }
+                        } else {
+                            if (_log.shouldLog(Log.INFO))
+                                _log.info("inbound: "+outmsg);
+                        }
+                        outmsg=outmsg+"\n";
+                        output.write(outmsg.getBytes());
+                    } else {
+                        if (_log.shouldLog(Log.WARN))
+                            _log.warn("inbound BLOCKED: "+inmsg);
+                    }
+                } catch (IOException e1) {
+                    if (_log.shouldLog(Log.ERROR))
+                        _log.error("IrcInboundFilter: disconnected",e1);
+                    break;
+                }
+            }
+            try {
+                local.close();
+            } catch (IOException e) {
+            }
+            if(_log.shouldLog(Log.DEBUG))
+                _log.debug("IrcInboundFilter: Done.");
+            }
+            
+        }
+                
+        /*************************************************************************
+         *
+         */
+        private class IrcOutboundFilter implements Runnable {
+                    
+            private Socket local;
+            private I2PSocket remote;
+                
+            IrcOutboundFilter(Socket _local, I2PSocket _remote) {
+                local=_local;
+                remote=_remote;
+            }
+                
+            public void run() {
+                InputStream input;
+                OutputStream output;
+                try {
+                    input=local.getInputStream();
+                    output=remote.getOutputStream();
+                } catch (IOException e) {
+                    if (_log.shouldLog(Log.ERROR))
+                        _log.error("IrcOutboundFilter: no streams",e);
+                    return;
+                }
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("IrcOutboundFilter: Running.");
+                while(true)
+                {
+                    try {
+                        String inmsg = DataHelper.readLine(input);
+                        if(inmsg==null)
+                            break;
+                        if(inmsg.endsWith("\r"))
+                            inmsg=inmsg.substring(0,inmsg.length()-1);
+                        String outmsg = outboundFilter(inmsg);
+                        if(outmsg!=null)
+                        {
+                            if(!inmsg.equals(outmsg)) {
+                                if (_log.shouldLog(Log.WARN)) {
+                                    _log.warn("outbound FILTERED: "+outmsg);
+                                    _log.warn(" - outbound was: "+inmsg);
+                                }
+                            } else {
+                                if (_log.shouldLog(Log.INFO))
+                                    _log.info("outbound: "+outmsg);
+                            }
+                            outmsg=outmsg+"\n";
+                            output.write(outmsg.getBytes());
+                        } else {
+                            if (_log.shouldLog(Log.WARN))
+                                _log.warn("outbound BLOCKED: "+"\""+inmsg+"\"");
+                        }
+                    } catch (IOException e1) {
+                        if (_log.shouldLog(Log.ERROR))
+                            _log.error("IrcOutboundFilter: disconnected",e1);
+                        break;
+                    }
+                }
+                try {
+                    remote.close();
+                } catch (IOException e) {
+                }
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("IrcOutboundFilter: Done.");
+            }
+        }
+
+    
+    /*************************************************************************
+     *
+     */
+    
+    public static String inboundFilter(String s) {
+        
+        String field[]=s.split(" ",4);
+        String command;
+        int idx=0;
+        final String[] allowedCommands =
+        {
+                "NOTICE",
+                "PING",
+                "PONG",
+                "MODE",
+                "JOIN",
+                "NICK",
+                "QUIT",
+                "PART",
+                "WALLOPS",
+                "ERROR"
+        };
+        
+        if(field[0].charAt(0)==':')
+            idx++;
+        
+        command = field[idx++];
+        
+        idx++; //skip victim
+
+        // Allow numerical responses
+        try {
+            new Integer(command);
+            return s;
+        } catch(NumberFormatException nfe){}
+        
+        // Allow all allowedCommands
+        for(int i=0;i<allowedCommands.length;i++) {
+            if(allowedCommands[i].equals(command))
+                return s;
+        }
+        
+        // Allow PRIVMSG, but block CTCP.
+        if("PRIVMSG".equals(command))
+        {
+            String msg;
+            msg = field[idx++];
+        
+            byte[] bytes = msg.getBytes();
+            if(bytes[1]==0x01)
+            {
+                // CTCP
+                msg=msg.substring(2);
+                if(msg.startsWith("ACTION ")) {
+                    // /me says hello
+                    return s;
+                }
+                return null; // Block all other ctcp
+            }
+            return s;
+        }
+        
+        // Block the rest
+        return null;
+    }
+    
+    public static String outboundFilter(String s) {
+        
+        String field[]=s.split(" ",3);
+        String command;
+        final String[] allowedCommands =
+        {
+                "NOTICE",
+                "PONG",
+                "MODE",
+                "JOIN",
+                "NICK",
+                "WHO",
+                "WHOIS",
+                "LIST",
+                "NAMES",
+                "NICK",
+                // "QUIT", // replace with a filtered QUIT to hide client quit messages
+                "SILENCE",
+                "PART",
+                "OPER",
+                "PING",
+                "KICK",
+                "HELPME",
+                "RULES"
+        };
+        
+        if(field[0].charAt(0)==':')
+            return null; // wtf
+        
+        command = field[0].toUpperCase();
+
+        // Allow all allowedCommands
+        for(int i=0;i<allowedCommands.length;i++)
+        {
+            if(allowedCommands[i].equals(command))
+                return s;
+        }
+        
+        // Allow PRIVMSG, but block CTCP (except ACTION).
+        if("PRIVMSG".equals(command))
+        {
+            String msg;
+            msg = field[2];
+        
+            byte[] bytes = msg.getBytes();
+            if(bytes[1]==0x01)
+            {
+                    // CTCP
+                msg=msg.substring(2);
+                if(msg.startsWith("ACTION ")) {
+                    // /me says hello
+                    return s;
+                }
+                return null; // Block all other ctcp
+            }
+            return s;
+        }
+        
+        if("USER".equals(command)) {
+            int idx = field[2].lastIndexOf(":");
+            if(idx<0)
+                return "USER user hostname localhost :realname";
+            String realname = field[2].substring(idx+1);
+            String ret = "USER "+field[1]+" hostname localhost :"+realname;
+            return ret;
+        } else if ("QUIT".equals(command)) {
+            return "QUIT :leaving";
+        }
+        
+        // Block the rest
+        return null;
+    }
+}

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java

@@ -135,6 +135,8 @@ public class TunnelController implements Logging {
         }
         if ("httpclient".equals(type)) {
             startHttpClient();
+        }else if("ircclient".equals(type)) {
+            startIrcClient();
         } else if ("client".equals(type)) {
             startClient();
         } else if ("server".equals(type)) {
@@ -162,6 +164,18 @@ public class TunnelController implements Logging {
         _running = true;
     }
     
+    private void startIrcClient() {
+        setI2CPOptions();
+        setSessionOptions();
+        setListenOn();
+        String listenPort = getListenPort();
+        String dest = getTargetDestination();
+        String sharedClient = getSharedClient();
+        _tunnel.runIrcClient(new String[] { listenPort, dest, sharedClient }, this);
+        acquire();
+        _running = true;
+    }
+    
     /** 
      * Note the fact that we are using some sessions, so that they dont get
      * closed by some other tunnels

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java

@@ -30,7 +30,9 @@ public class EditBean extends IndexBean {
         if (controllers.size() > tunnel) {
             TunnelController cur = (TunnelController)controllers.get(tunnel);
             if (cur == null) return false;
-            return ( ("client".equals(cur.getType())) || ("httpclient".equals(cur.getType())) );
+            return ( ("client".equals(cur.getType())) || 
+            		 ("httpclient".equals(cur.getType()))||
+            		 ("ircclient".equals(cur.getType())));
         } else {
             return false;
         }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java

@@ -198,7 +198,9 @@ public class IndexBean {
             cur.setConfig(config, "");
         }
         
-        if ("httpclient".equals(cur.getType()) || "client".equals(cur.getType())) {
+        if ("ircclient".equals(cur.getType()) || 
+        		"httpclient".equals(cur.getType()) || 
+        		"client".equals(cur.getType())) {
             // all clients use the same I2CP session, and as such, use the same
             // I2CP options
             List controllers = _group.getControllers();
@@ -206,7 +208,10 @@ public class IndexBean {
                 TunnelController c = (TunnelController)controllers.get(i);
                 if (c == cur) continue;
                 //only change when they really are declared of beeing a sharedClient
-                if (("httpclient".equals(c.getType()) || "client".equals(c.getType())) && "true".equalsIgnoreCase(c.getSharedClient())) {
+                if (("httpclient".equals(c.getType()) || 
+                		"ircclient".equals(c.getType())||
+                		"client".equals(c.getType()) 
+                		) && "true".equalsIgnoreCase(c.getSharedClient())) {
                     Properties cOpt = c.getConfig("");
                     if (_tunnelCount != null) {
                         cOpt.setProperty("option.inbound.quantity", _tunnelCount);
@@ -278,7 +283,9 @@ public class IndexBean {
     public boolean isClient(int tunnelNum) {
         TunnelController cur = getController(tunnelNum);
         if (cur == null) return false;
-        return ( ("client".equals(cur.getType())) || ("httpclient".equals(cur.getType())) );
+        return ( ("client".equals(cur.getType())) || 
+        		("httpclient".equals(cur.getType())) ||
+        		("ircclient".equals(cur.getType())));
     }
     
     public String getTunnelName(int tunnel) {
@@ -308,6 +315,7 @@ public class IndexBean {
     public String getTypeName(String internalType) {
         if ("client".equals(internalType)) return "Client proxy";
         else if ("httpclient".equals(internalType)) return "HTTP proxy";
+        else if ("ircclient".equals(internalType)) return "IRC proxy";
         else if ("server".equals(internalType)) return "Server";
         else if ("httpserver".equals(internalType)) return "HTTP server";
         else return internalType;
@@ -356,7 +364,7 @@ public class IndexBean {
     public String getClientDestination(int tunnel) {
         TunnelController tun = getController(tunnel);
         if (tun == null) return "";
-        if ("client".equals(tun.getType())) return tun.getTargetDestination();
+        if ("client".equals(tun.getType())||"ircclient".equals(tun.getType())) return tun.getTargetDestination();
         else return tun.getProxyList();
     }
     
@@ -386,7 +394,7 @@ public class IndexBean {
     ///
     
     /**
-     * What type of tunnel (httpclient, client, or server).  This is 
+     * What type of tunnel (httpclient, ircclient, client, or server).  This is 
      * required when adding a new tunnel.
      *
      */
@@ -427,12 +435,12 @@ public class IndexBean {
     public void setProxyList(String proxyList) { 
         _proxyList = (proxyList != null ? proxyList.trim() : null);
     }
-    /** what port should this client/httpclient listen on */
+    /** what port should this client/httpclient/ircclient listen on */
     public void setPort(String port) { 
         _port = (port != null ? port.trim() : null);
     }
     /** 
-     * what interface should this client/httpclient listen on (unless 
+     * what interface should this client/httpclient/ircclient listen on (unless 
      * overridden by the setReachableByOther() field)
      */
     public void setReachableBy(String reachableBy) { 
@@ -440,7 +448,7 @@ public class IndexBean {
     }
     /**
      * If specified, defines the exact IP interface to listen for requests
-     * on (in the case of client/httpclient tunnels)
+     * on (in the case of client/httpclient/ircclient tunnels)
      */
     public void setReachableByOther(String reachableByOther) { 
         _reachableByOther = (reachableByOther != null ? reachableByOther.trim() : null);
@@ -520,6 +528,24 @@ public class IndexBean {
             }
 
             config.setProperty("sharedClient", _sharedClient + "");
+        }else if ("ircclient".equals(_type)) {
+                if (_port != null)
+                    config.setProperty("listenPort", _port);
+                if (_reachableByOther != null)
+                    config.setProperty("interface", _reachableByOther);
+                else
+                    config.setProperty("interface", _reachableBy);
+                if (_targetDestination != null)
+                    config.setProperty("targetDestination", _targetDestination);
+
+            	config.setProperty("option.inbound.nickname", CLIENT_NICKNAME);
+            	config.setProperty("option.outbound.nickname", CLIENT_NICKNAME);
+                if (_name != null && !_sharedClient) {
+                     config.setProperty("option.inbound.nickname", _name);
+                     config.setProperty("option.outbound.nickname", _name);
+                }
+
+                config.setProperty("sharedClient", _sharedClient + "");
         } else if ("client".equals(_type)) {
             if (_port != null)
                 config.setProperty("listenPort", _port);
@@ -608,7 +634,7 @@ public class IndexBean {
         else
             config.setProperty("option.i2p.streaming.connectDelay", "0");
         if (_name != null) {
-            if ( ((!"client".equals(_type)) && (!"httpclient".equals(_type))) || (!_sharedClient) ) {
+            if ( ((!"client".equals(_type)) && (!"httpclient".equals(_type))&& (!"ircclient".equals(_type))) || (!_sharedClient) ) {
                 config.setProperty("option.inbound.nickname", _name);
                 config.setProperty("option.outbound.nickname", _name);
             } else {

apps/i2ptunnel/jsp/edit.jsp

@@ -15,7 +15,7 @@
   } else {
     String type = request.getParameter("type");
     int curTunnel = -1;
-    if ("client".equals(type) || "httpclient".equals(type)) {
+    if ("client".equals(type) || "httpclient".equals(type) || "ircclient".equals(type)) {
       %><jsp:include page="editClient.jsp" /><%
     } else if ("server".equals(type) || "httpserver".equals(type)) {
       %><jsp:include page="editServer.jsp" /><%

apps/i2ptunnel/jsp/editClient.jsp

@@ -170,7 +170,7 @@ if (curTunnel >= 0) {
 <% } else { %>
 <input type="checkbox" value="true" name="shared" />
 <% } %>
-<i>(Share tunnels with other clients and httpclients? Change requires restart of client proxy)</i>
+<i>(Share tunnels with other clients and irc/httpclients? Change requires restart of client proxy)</i>
 </td>
 </tr>
 <tr>

apps/i2ptunnel/jsp/index.jsp

@@ -169,6 +169,7 @@
 <b>Add new:</b> 
 <select name="type">
   <option value="httpclient">HTTP proxy</option>
+  <option value="ircclient">IRC proxy</option>
   <option value="client">Client tunnel</option>
   <option value="server">Server tunnel</option>
    <option value="httpserver">HTTP server tunnel</option>

apps/jdom/readme.txt

@@ -0,0 +1 @@
+This is JDOM 1.0 from http://jdom.org/, released under an Apache style license

apps/rome/readme.txt

@@ -0,0 +1 @@
+This is ROME 0.7 from http://rome.dev.java.net/, released under a BSD license

apps/streaming/java/src/net/i2p/client/streaming/Connection.java

@@ -732,10 +732,12 @@ public class Connection {
                 _log.debug("Resetting the inactivity timer, but its gone!", new Exception("where did it go?"));
             return;
         }
-        long howLong = _activityTimer.getTimeLeft();
+        long howLong = _options.getInactivityTimeout();
+        howLong += _context.random().nextInt(30*1000); // randomize it a bit, so both sides don't do it at once
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Resetting the inactivity timer to " + howLong, new Exception("Reset by"));
         // this will get rescheduled, and rescheduled, and rescheduled...
+        SimpleTimer.getInstance().removeEvent(_activityTimer);
         SimpleTimer.getInstance().addEvent(_activityTimer, howLong);
     }
     

apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java

@@ -105,8 +105,8 @@ public class ConnectionOptions extends I2PSocketOptionsImpl {
         setWindowSize(getInt(opts, PROP_INITIAL_WINDOW_SIZE, INITIAL_WINDOW_SIZE));
         setMaxResends(getInt(opts, PROP_MAX_RESENDS, DEFAULT_MAX_SENDS));
         setWriteTimeout(getInt(opts, PROP_WRITE_TIMEOUT, -1));
-        setInactivityTimeout(getInt(opts, PROP_INACTIVITY_TIMEOUT, 2*60*1000));
-        setInactivityAction(getInt(opts, PROP_INACTIVITY_ACTION, INACTIVITY_ACTION_DISCONNECT));
+        setInactivityTimeout(getInt(opts, PROP_INACTIVITY_TIMEOUT, 90*1000));
+        setInactivityAction(getInt(opts, PROP_INACTIVITY_ACTION, INACTIVITY_ACTION_SEND));
         setInboundBufferSize(getMaxMessageSize() * (Connection.MAX_WINDOW_SIZE + 2));
         setCongestionAvoidanceGrowthRateFactor(getInt(opts, PROP_CONGESTION_AVOIDANCE_GROWTH_RATE_FACTOR, 1));
         setSlowStartGrowthRateFactor(getInt(opts, PROP_SLOW_START_GROWTH_RATE_FACTOR, 1));
@@ -140,9 +140,9 @@ public class ConnectionOptions extends I2PSocketOptionsImpl {
         if (opts.containsKey(PROP_WRITE_TIMEOUT))
             setWriteTimeout(getInt(opts, PROP_WRITE_TIMEOUT, -1));
         if (opts.containsKey(PROP_INACTIVITY_TIMEOUT))
-            setInactivityTimeout(getInt(opts, PROP_INACTIVITY_TIMEOUT, 2*60*1000));
+            setInactivityTimeout(getInt(opts, PROP_INACTIVITY_TIMEOUT, 90*1000));
         if (opts.containsKey(PROP_INACTIVITY_ACTION))
-            setInactivityAction(getInt(opts, PROP_INACTIVITY_ACTION, INACTIVITY_ACTION_DISCONNECT));
+            setInactivityAction(getInt(opts, PROP_INACTIVITY_ACTION, INACTIVITY_ACTION_SEND));
         setInboundBufferSize(getMaxMessageSize() * (Connection.MAX_WINDOW_SIZE + 2));
         if (opts.contains(PROP_CONGESTION_AVOIDANCE_GROWTH_RATE_FACTOR))
             setCongestionAvoidanceGrowthRateFactor(getInt(opts, PROP_CONGESTION_AVOIDANCE_GROWTH_RATE_FACTOR, 2));

apps/syndie/java/build.xml

@@ -14,15 +14,21 @@
             srcdir="./src" 
             debug="true" deprecation="on" source="1.3" target="1.3" 
             destdir="./build/obj" 
-            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar" />
+            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jdom/jdom.jar:../../rome/rome-0.7.jar" />
     </target>
     <target name="jar" depends="builddep, compile">
         <jar destfile="./build/syndie.jar" basedir="./build/obj" includes="**/*.class">
             <manifest>
-                <attribute name="Main-Class" value="net.i2p.syndie.CLI" />
+                <attribute name="Main-Class" value="net.i2p.syndie.CLIPost" />
                 <attribute name="Class-Path" value="i2p.jar" />
             </manifest>
         </jar>
+        <jar destfile="./build/sucker.jar" basedir="./build/obj" includes="net/i2p/syndie/Sucker.class, net/i2p/syndie/SuckerFetchListener.class">
+            <manifest>
+                <attribute name="Main-Class" value="net.i2p.syndie.Sucker" />
+                <attribute name="Class-Path" value="i2p.jar jdom.jar rome-0.7.jar" />
+            </manifest>
+        </jar>
         <ant target="war" />
     </target>
     <target name="war" depends="builddep, compile, precompilejsp"> 
@@ -48,6 +54,8 @@
                 <pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
                 <pathelement location="../../jetty/jettylib/javax.servlet.jar" />
                 <pathelement location="../../jetty/jettylib/ant.jar" />
+                <pathelement location="../../jdom/jdom.jar" />
+                <pathelement location="../../rome/rome-0.7.jar" />
                 <pathelement location="build/obj" />
                 <pathelement location="../../../core/java/build/i2p.jar" />
             </classpath>
@@ -69,6 +77,8 @@
                 <pathelement location="../../jetty/jettylib/commons-el.jar" />
                 <pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
                 <pathelement location="../../jetty/jettylib/javax.servlet.jar" />
+                <pathelement location="../../jdom/jdom.jar" />
+                <pathelement location="../../rome/rome-0.7.jar" />
                 <pathelement location="build/obj" />
                 <pathelement location="../../../core/java/build/i2p.jar" />
             </classpath>

apps/syndie/java/src/net/i2p/syndie/Archive.java

@@ -6,6 +6,7 @@ import java.text.*;
 import net.i2p.I2PAppContext;
 import net.i2p.data.*;
 import net.i2p.syndie.data.*;
+import net.i2p.util.Log;
 
 /**
  * Store blog info in the local filesystem.
@@ -25,6 +26,7 @@ import net.i2p.syndie.data.*;
  */
 public class Archive {
     private I2PAppContext _context;
+    private Log _log;
     private File _rootDir;
     private File _cacheDir;
     private Map _blogInfo;
@@ -42,6 +44,7 @@ public class Archive {
     
     public Archive(I2PAppContext ctx, String rootDir, String cacheDir) {
         _context = ctx;
+        _log = ctx.logManager().getLog(Archive.class);
         _rootDir = new File(rootDir);
         if (!_rootDir.exists())
             _rootDir.mkdirs();
@@ -69,12 +72,11 @@ public class Archive {
                         if (bi.verify(_context)) {
                             info.add(bi);
                         } else {
-                            System.err.println("Invalid blog (but we're storing it anyway): " + bi);
-                            new Exception("foo").printStackTrace();
-                            info.add(bi);
+                            _log.error("BlogInfo is invalid: " + bi);
+                            meta.delete();
                         }
                     } catch (IOException ioe) {
-                        ioe.printStackTrace();
+                        _log.error("Error loading the blog", ioe);
                     }
                 }
             }
@@ -110,8 +112,7 @@ public class Archive {
     }
     public boolean storeBlogInfo(BlogInfo info) { 
         if (!info.verify(_context)) {
-            System.err.println("Not storing the invalid blog " + info);
-            new Exception("foo!").printStackTrace();
+            _log.warn("Not storing invalid blog " + info);
             return false;
         }
         boolean isNew = true;
@@ -130,10 +131,11 @@ public class Archive {
             FileOutputStream out = new FileOutputStream(blogFile);
             info.write(out);
             out.close();
-            System.out.println("Blog info written to " + blogFile.getPath());
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Blog info written to " + blogFile.getPath());
             return true;
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error writing out info", ioe);
             return false;
         }
     }
@@ -174,8 +176,9 @@ public class Archive {
                     entry = getCachedEntry(entryDir);
                 if ( (entry == null) || (!entryDir.exists()) ) {
                     if (!extractEntry(entries[j], entryDir, info)) {
-                        System.err.println("Entry " + entries[j].getPath() + " is not valid");
-                        new Exception("foo!!").printStackTrace();
+                        if (_log.shouldLog(Log.ERROR))
+                            _log.error("Entry " + entries[j].getPath() + " is not valid");
+                        entries[j].delete();
                         continue;
                     }
                     entry = getCachedEntry(entryDir);
@@ -183,12 +186,13 @@ public class Archive {
                 String tags[] = entry.getTags();
                 for (int t = 0; t < tags.length; t++) {
                     if (!rv.contains(tags[t])) {
-                        System.out.println("Found a new tag in cached " + entry.getURI() + ": " + tags[t]);
+                        if (_log.shouldLog(Log.DEBUG))
+                            _log.debug("Found a new tag in cached " + entry.getURI() + ": " + tags[t]);
                         rv.add(tags[t]);
                     }
                 }
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                _log.error("Error listing tags", ioe);
             }
         } // end iterating over the entries 
         
@@ -220,7 +224,7 @@ public class Archive {
                 return ce;
             return null;
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.warn("Error reading cached entry... deleting cache elements");
         }
         
         File files[] = entryDir.listFiles();
@@ -262,8 +266,8 @@ public class Archive {
                         entry = getCachedEntry(entryDir);
                     if ((entry == null) || !entryDir.exists()) {
                         if (!extractEntry(entries[i], entryDir, info)) {
-                            System.err.println("Entry " + entries[i].getPath() + " is not valid");
-                            new Exception("foo!!!!").printStackTrace();
+                            _log.error("Entry " + entries[i].getPath() + " is not valid");
+                            entries[i].delete();
                             continue;
                         }
                         entry = getCachedEntry(entryDir);
@@ -274,8 +278,8 @@ public class Archive {
                     entry.load(new FileInputStream(entries[i]));
                     boolean ok = entry.verifySignature(_context, info);
                     if (!ok) {
-                        System.err.println("Keyed entry " + entries[i].getPath() + " is not valid");
-                        new Exception("foo!!!!!!").printStackTrace();
+                        _log.error("Keyed entry " + entries[i].getPath() + " is not valid");
+                        entries[i].delete();
                         continue;
                     }
 
@@ -294,16 +298,18 @@ public class Archive {
                     for (int j = 0; j < tags.length; j++) {
                         if (tags[j].equals(tag)) {
                             rv.add(entry);
-                            System.out.println("cached entry matched requested tag [" + tag + "]: " + entry.getURI());
+                            if (_log.shouldLog(Log.DEBUG))
+                                _log.debug("cached entry matched requested tag [" + tag + "]: " + entry.getURI());
                             break;
                         }
                     }
                 } else {
-                    System.out.println("cached entry is ok and no id or tag was requested: " + entry.getURI());
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("cached entry is ok and no id or tag was requested: " + entry.getURI());
                     rv.add(entry);
                 }
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                _log.error("Error listing entries", ioe);
             }
         }
         return rv;
@@ -322,11 +328,11 @@ public class Archive {
 
         BlogInfo info = getBlogInfo(uri);
         if (info == null) {
-            System.out.println("no blog metadata for the uri " + uri);
+            _log.error("no blog metadata for the uri " + uri);
             return false;
         }
         if (!container.verifySignature(_context, info)) {
-            System.out.println("Not storing the invalid blog entry at " + uri);
+            _log.error("Not storing the invalid blog entry at " + uri);
             return false;
         } else {
             //System.out.println("Signature is valid: " + container.getSignature() + " for info " + info);
@@ -341,7 +347,7 @@ public class Archive {
             container.setCompleteSize(data.length);
             return true;
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error storing", ioe);
             return false;
         }
     }
@@ -422,7 +428,7 @@ public class Archive {
             out.write(DataHelper.getUTF8(_index.toString()));
             out.flush();
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error writing out the index");
         }
     }
 }

apps/syndie/java/src/net/i2p/syndie/ArchiveIndexer.java

@@ -7,6 +7,7 @@ import net.i2p.I2PAppContext;
 import net.i2p.data.*;
 import net.i2p.syndie.data.*;
 import net.i2p.syndie.sml.*;
+import net.i2p.util.Log;
 
 /**
  * Dig through the archive to build an index
@@ -16,6 +17,7 @@ class ArchiveIndexer {
     private static final int RECENT_ENTRY_COUNT = 10;
     
     public static ArchiveIndex index(I2PAppContext ctx, Archive source) {
+        Log log = ctx.logManager().getLog(ArchiveIndexer.class);
         LocalArchiveIndex rv = new LocalArchiveIndex(ctx);
         rv.setGeneratedOn(ctx.clock().now());
         
@@ -32,7 +34,7 @@ class ArchiveIndexer {
                         rv.setHeader(tok.nextToken(), tok.nextToken());
                 }
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                log.error("Error reading header file", ioe);
             }
         }
         
@@ -66,7 +68,8 @@ class ArchiveIndexer {
             long metadate = metaFile.lastModified();
 
             List entries = source.listEntries(key, -1, null, null);
-            System.out.println("Entries under " + key + ": " + entries);
+            if (log.shouldLog(Log.DEBUG))
+                log.debug("Entries under " + key + ": " + entries);
             /** tag name --> ordered map of entryId to EntryContainer */
             Map tags = new TreeMap();
             
@@ -83,7 +86,8 @@ class ArchiveIndexer {
                     }
                     Map entriesByTag = (Map)tags.get(entryTags[t]);
                     entriesByTag.put(new Long(0-entry.getURI().getEntryId()), entry);
-                    System.out.println("Entries under tag " + entryTags[t] + ":" + entriesByTag.values());
+                    if (log.shouldLog(Log.DEBUG))
+                        log.debug("Entries under tag " + entryTags[t] + ":" + entriesByTag.values());
                 }
                     
                 if (entry.getURI().getEntryId() >= newSince) {
@@ -97,8 +101,8 @@ class ArchiveIndexer {
                     BlogURI parent = new BlogURI(reply.trim());
                     if ( (parent.getKeyHash() != null) && (parent.getEntryId() >= 0) ) 
                         rv.addReply(parent, entry.getURI());
-                    else
-                        System.err.println("Parent of " + entry.getURI() + " is not valid: [" + reply.trim() + "]");
+                    else if (log.shouldLog(Log.WARN))
+                        log.warn("Parent of " + entry.getURI() + " is not valid: [" + reply.trim() + "]");
                 }
             }
             

apps/syndie/java/src/net/i2p/syndie/BlogManager.java

@@ -10,12 +10,14 @@ import net.i2p.client.naming.PetNameDB;
 import net.i2p.data.*;
 import net.i2p.syndie.data.*;
 import net.i2p.syndie.sml.*;
+import net.i2p.util.Log;
 
 /**
  *
  */
 public class BlogManager {
     private I2PAppContext _context;
+    private Log _log;
     private static BlogManager _instance;
     private File _blogKeyDir;
     private File _privKeyDir;
@@ -28,21 +30,30 @@ public class BlogManager {
     
     static {
         TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
-        String rootDir = I2PAppContext.getGlobalContext().getProperty("syndie.rootDir");
-        if (false) {
-            if (rootDir == null)
-                rootDir = System.getProperty("user.home");
-            rootDir = rootDir + File.separatorChar + ".syndie";
-        } else {
-            if (rootDir == null)
-                rootDir = "./syndie";
-        }
-        _instance = new BlogManager(I2PAppContext.getGlobalContext(), rootDir);
     }
-    public static BlogManager instance() { return _instance; }
     
-    public BlogManager(I2PAppContext ctx, String rootDir) {
+    public static BlogManager instance() { 
+        synchronized (BlogManager.class) {
+            if (_instance == null) {
+                String rootDir = I2PAppContext.getGlobalContext().getProperty("syndie.rootDir");
+                if (false) {
+                    if (rootDir == null)
+                        rootDir = System.getProperty("user.home");
+                    rootDir = rootDir + File.separatorChar + ".syndie";
+                } else {
+                    if (rootDir == null)
+                        rootDir = "./syndie";
+                }
+                _instance = new BlogManager(I2PAppContext.getGlobalContext(), rootDir);
+            }
+            return _instance; 
+        }
+    }
+    
+    public BlogManager(I2PAppContext ctx, String rootDir) { this(ctx, rootDir, true); }
+    public BlogManager(I2PAppContext ctx, String rootDir, boolean regenIndex) {
         _context = ctx;
+        _log = ctx.logManager().getLog(BlogManager.class);
         _rootDir = new File(rootDir);
         _rootDir.mkdirs();
         readConfig();
@@ -63,7 +74,8 @@ public class BlogManager {
         _userDir.mkdirs();
         _tempDir.mkdirs();
         _archive = new Archive(ctx, _archiveDir.getAbsolutePath(), _cacheDir.getAbsolutePath());
-        _archive.regenerateIndex();
+        if (regenIndex)
+            _archive.regenerateIndex();
     }
     
     private File getConfigFile() { return new File(_rootDir, "syndie.config"); }
@@ -76,13 +88,16 @@ public class BlogManager {
                 for (Iterator iter = p.keySet().iterator(); iter.hasNext(); ) {
                     String key = (String)iter.next();
                     System.setProperty(key, p.getProperty(key));
-                    System.out.println("Read config prop [" + key + "] = [" + p.getProperty(key) + "]");
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Read config prop [" + key + "] = [" + p.getProperty(key) + "]");
                 }
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Err reading", ioe);
             }
         } else {
-            System.out.println("Config doesn't exist: " + config.getPath());
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Config doesn't exist: " + config.getPath());
         }
     }
     
@@ -97,7 +112,7 @@ public class BlogManager {
                     out.write(DataHelper.getUTF8(name + '=' + _context.getProperty(name) + '\n'));
             }
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error writing the config", ioe);
         } finally {
             if (out != null) try { out.close(); } catch (IOException ioe) {}
         }
@@ -116,10 +131,10 @@ public class BlogManager {
             pub.writeBytes(out);
             priv.writeBytes(out);
         } catch (DataFormatException dfe) {
-            dfe.printStackTrace();
+            _log.error("Error creating the blog", dfe);
             return null;
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error creating the blog", ioe);
             return null;
         }
         
@@ -181,9 +196,9 @@ public class BlogManager {
                     if (info != null)
                         rv.add(info);
                 } catch (IOException ioe) {
-                    ioe.printStackTrace();
+                    _log.error("Error listing the blog", ioe);
                 } catch (DataFormatException dfe) {
-                    dfe.printStackTrace();
+                    _log.error("Error listing the blog", dfe);
                 }
             }
         }
@@ -201,10 +216,66 @@ public class BlogManager {
             priv.readBytes(in);
             return priv;
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error reading the blog key", ioe);
             return null;
         } catch (DataFormatException dfe) {
-            dfe.printStackTrace();
+            _log.error("Error reading the blog key", dfe);
+            return null;
+        }
+    }
+    
+    public User getUser(Hash blog) {
+        File files[] = _userDir.listFiles();
+        for (int i = 0; i < files.length; i++) {
+            if (files[i].isFile() && !files[i].isHidden()) {
+                Properties userProps = loadUserProps(files[i]);
+                if (userProps == null)
+                    continue;
+                User user = new User();
+                user.load(userProps);
+                if (blog.equals(user.getBlog()))
+                    return user;
+            }
+        }
+        return null;
+    }
+    
+    /**
+     * List of User instances
+     */
+    public List listUsers() {
+        File files[] = _userDir.listFiles();
+        List rv = new ArrayList();
+        for (int i = 0; i < files.length; i++) {
+            if (files[i].isFile() && !files[i].isHidden()) {
+                Properties userProps = loadUserProps(files[i]);
+                if (userProps == null)
+                    continue;
+                User user = new User();
+                user.load(userProps);
+                rv.add(user);
+            }
+        }
+        return rv;
+    }
+    
+    private Properties loadUserProps(File userFile) {
+        try {
+            Properties props = new Properties();
+            FileInputStream fin = new FileInputStream(userFile);
+            BufferedReader in = new BufferedReader(new InputStreamReader(fin, "UTF-8"));
+            String line = null;
+            while ( (line = in.readLine()) != null) {
+                int split = line.indexOf('=');
+                if (split <= 0) continue;
+                String key = line.substring(0, split);
+                String val = line.substring(split+1);
+                props.setProperty(key.trim(), val.trim());
+            }
+            String userHash = userFile.getName();
+            props.setProperty(User.PROP_USERHASH, userHash);
+            return props;
+        } catch (IOException ioe) {
             return null;
         }
     }
@@ -214,25 +285,17 @@ public class BlogManager {
         Hash userHash = _context.sha().calculateHash(DataHelper.getUTF8(login));
         Hash passHash = _context.sha().calculateHash(DataHelper.getUTF8(pass));
         File userFile = new File(_userDir, Base64.encode(userHash.getData()));
-        System.out.println("Attempting to login to " + login + " w/ pass = " + pass 
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Attempting to login to " + login + " w/ pass = " + pass 
                            + ": file = " + userFile.getAbsolutePath() + " passHash = "
                            + Base64.encode(passHash.getData()));
         if (userFile.exists()) {
             try {
-                Properties props = new Properties();
-                FileInputStream fin = new FileInputStream(userFile);
-                BufferedReader in = new BufferedReader(new InputStreamReader(fin, "UTF-8"));
-                String line = null;
-                while ( (line = in.readLine()) != null) {
-                    int split = line.indexOf('=');
-                    if (split <= 0) continue;
-                    String key = line.substring(0, split);
-                    String val = line.substring(split+1);
-                    props.setProperty(key.trim(), val.trim());
-                }
+                Properties props = loadUserProps(userFile);
+                if (props == null) throw new IOException("Error reading " + userFile);
                 return user.login(login, pass, props);
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                _log.error("Error logging in", ioe);
                 return "<span class=\"b_loginMsgErr\">Error logging in - corrupt userfile</span>";
             }
         } else {
@@ -251,7 +314,6 @@ public class BlogManager {
     public String getRemotePasswordHash() { 
         String pass = _context.getProperty("syndie.remotePassword");
         
-        System.out.println("Remote password? [" + pass + "]");
         if ( (pass == null) || (pass.trim().length() <= 0) ) return null;
         return pass;
     }
@@ -279,7 +341,8 @@ public class BlogManager {
     public String getDefaultProxyHost() { return _context.getProperty("syndie.defaultProxyHost", ""); }
     public String getDefaultProxyPort() { return _context.getProperty("syndie.defaultProxyPort", ""); }
     public int getUpdateDelay() { return Integer.parseInt(_context.getProperty("syndie.updateDelay", "12")); }
-    public String[] getUpdateArchives() { return _context.getProperty("syndie.updateArchives", "").split(","); } 
+    public String[] getUpdateArchives() { return _context.getProperty("syndie.updateArchives", "").split(","); }
+    public boolean getImportAddresses() { return _context.getProperty("syndie.importAddresses", "false").equals("true"); }
     
     public boolean authorizeAdmin(String pass) {
         if (isSingleUser()) return true;
@@ -330,7 +393,7 @@ public class BlogManager {
             }
             _archive.setDefaultSelector(defaultSelector);
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error writing out the config", ioe);
         } finally {
             if (out != null) try { out.close(); } catch (IOException ioe) {}
             readConfig();
@@ -353,21 +416,30 @@ public class BlogManager {
         }
     }
     
-    public void saveUser(User user) {
-        if (!user.getAuthenticated()) return;
-        String userHash = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(user.getUsername())).getData());
+    /**
+     * Store user info, regardless of whether they're logged in.  This lets you update a
+     * different user's info!
+     */
+    void storeUser(User user) {
+        String userHash = user.getUserHash();
         File userFile = new File(_userDir, userHash);
+        if (!userFile.exists()) return;
         FileOutputStream out = null;
         try {
             out = new FileOutputStream(userFile);
             out.write(DataHelper.getUTF8(user.export()));
             user.getPetNameDB().store(user.getAddressbookLocation());
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error writing out the user", ioe);
         } finally {
             if (out != null) try { out.close(); } catch (IOException ioe){}
         }
     }
+    
+    public void saveUser(User user) {
+        if (!user.getAuthenticated()) return;
+        storeUser(user);
+    }
     public String register(User user, String login, String password, String registrationPassword, String blogName, String blogDescription, String contactURL) {
         System.err.println("Register [" + login + "] pass [" + password + "] name [" + blogName + "] descr [" + blogDescription + "] contact [" + contactURL + "] regPass [" + registrationPassword + "]");
         String hashedRegistrationPassword = getRegistrationPasswordHash();
@@ -399,7 +471,7 @@ public class BlogManager {
                 bw.write("showexpanded=false\n");
                 bw.flush();
             } catch (IOException ioe) {
-                ioe.printStackTrace();
+                _log.error("Error registering the user", ioe);
                 return "<span class=\"b_regMsgErr\">Internal error registering - " + ioe.getMessage() + "</span>";
             } finally {
                 if (out != null) try { out.close(); } catch (IOException ioe) {}
@@ -416,16 +488,16 @@ public class BlogManager {
         PetNameDB userDb = user.getPetNameDB();
         PetNameDB routerDb = _context.petnameDb();
         // horribly inefficient...
-        for (Iterator names = userDb.getNames().iterator(); names.hasNext();) {
-            PetName pn = userDb.get((String)names.next());
+        for (Iterator iter = userDb.iterator(); iter.hasNext();) {
+            PetName pn = (PetName)iter.next();
             if (pn == null) continue;
             Destination existing = _context.namingService().lookup(pn.getName());
             if (existing == null && pn.getNetwork().equalsIgnoreCase("i2p")) {
-                routerDb.set(pn.getName(), pn);
+                routerDb.add(pn);
                 try {
                     routerDb.store();
                 } catch (IOException ioe) {
-                    ioe.printStackTrace();
+                    _log.error("Error exporting the hosts", ioe);
                     return "<span class=\"b_addrMsgErr\">Error exporting the hosts: " + ioe.getMessage() + "</span>";
                 }
             }
@@ -433,6 +505,21 @@ public class BlogManager {
         return "<span class=\"b_addrMsgOk\">Hosts exported</span>";
     }
     
+    /**
+     * Guess what the next entry ID should be for the given user.  Rounds down to 
+     * midnight of the current day + 1 for each post in that day.
+     */
+    public long getNextBlogEntry(User user) {
+        long entryId = -1;
+        long now = _context.clock().now();
+        long dayBegin = getDayBegin(now);
+        if (user.getMostRecentEntry() >= dayBegin)
+            entryId = user.getMostRecentEntry() + 1;
+        else
+            entryId = dayBegin;
+        return entryId;
+    }
+    
     public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml) {
         return createBlogEntry(user, subject, tags, entryHeaders, sml, null, null, null);
     }
@@ -443,13 +530,7 @@ public class BlogManager {
         SigningPrivateKey privkey = getMyPrivateKey(info);
         if (privkey == null) return null;
         
-        long entryId = -1;
-        long now = _context.clock().now();
-        long dayBegin = getDayBegin(now);
-        if (user.getMostRecentEntry() >= dayBegin)
-            entryId = user.getMostRecentEntry() + 1;
-        else
-            entryId = dayBegin;
+        long entryId = getNextBlogEntry(user);
         
         StringTokenizer tok = new StringTokenizer(tags, " ,\n\t");
         String tagList[] = new String[tok.countTokens()];
@@ -466,12 +547,14 @@ public class BlogManager {
                 raw.append(tagList[i]).append('\t');
             raw.append('\n');
             if ( (entryHeaders != null) && (entryHeaders.trim().length() > 0) ) {
-                System.out.println("Entry headers: " + entryHeaders);
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Creating entry with headers: " + entryHeaders);
                 BufferedReader userHeaders = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(DataHelper.getUTF8(entryHeaders)), "UTF-8"));
                 String line = null;
                 while ( (line = userHeaders.readLine()) != null) {
                     line = line.trim();
-                    System.out.println("Line: " + line);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("header line: " + line);
                     if (line.length() <= 0) continue;
                     int split = line.indexOf('=');
                     int split2 = line.indexOf(':');
@@ -520,7 +603,7 @@ public class BlogManager {
                 return null;
             }
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error creating post", ioe);
             return null;
         }
     }
@@ -536,7 +619,7 @@ public class BlogManager {
             info.load(metadataStream);
             return _archive.storeBlogInfo(info);
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error importing meta", ioe);
             return false;
         }
     }
@@ -552,7 +635,7 @@ public class BlogManager {
             c.load(entryStream);
             return _archive.storeEntry(c);
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("Error importing entry", ioe);
             return false;
         }
     }
@@ -567,10 +650,10 @@ public class BlogManager {
         // no need to quote user/location further, as they've been sanitized
         
         PetNameDB names = user.getPetNameDB();
-        if (names.exists(name))
+        if (names.containsName(name))
             return "<span class=\"b_addrMsgErr\">Name is already in use</span>";
         PetName pn = new PetName(name, schema, protocol, location);
-        names.set(name, pn);
+        names.add(pn);
         
         try {
             names.store(user.getAddressbookLocation());
@@ -616,7 +699,7 @@ public class BlogManager {
                 Destination d = new Destination(location);
                 return (d.getPublicKey() != null);
             } catch (DataFormatException dfe) {
-                dfe.printStackTrace();
+                _log.error("Error validating address location", dfe);
                 return false;
             }
         } else {

apps/syndie/java/src/net/i2p/syndie/CLI.java

@@ -13,7 +13,7 @@ public class CLI {
     public static final String USAGE = "Usage: \n" +
         "rootDir regenerateIndex\n" +
         "rootDir createBlog name description contactURL[ archiveURL]*\n" +
-        "rootDir createEntry blogPublicKeyHash tag[,tag]* (NOW|entryId) (NONE|entryKeyBase64) smlFile[ attachmentFile]*\n" +
+        "rootDir createEntry blogPublicKeyHash tag[,tag]* (NEXT|NOW|entryId) (NONE|entryKeyBase64) smlFile[ attachmentFile attachmentName attachmentDescription mimeType]*\n" +
         "rootDir listMyBlogs\n" +
         "rootDir listTags blogPublicKeyHash\n" +
         "rootDir listEntries blogPublicKeyHash blogTag\n" +
@@ -130,50 +130,117 @@ public class CLI {
     }
 
     private static void createEntry(String args[]) {
-        // "rootDir createEntry blogPublicKey tag[,tag]* (NOW|entryId) (NONE|entryKeyBase64) smlFile[ attachmentFile]*\n" +
-
+        // "rootDir createEntry blogPublicKeyHash tag[,tag]* (NEXT|NOW|entryId) (NONE|entryKeyBase64) "
+        //  smlFile[ attachmentFile attachmentName attachmentDescription mimeType]*\n"
+        String rootDir = args[0];
+        String hashStr = args[2];
+        List tags = new ArrayList();
+        StringTokenizer tok = new StringTokenizer(args[3], ",");
+        while (tok.hasMoreTokens())
+            tags.add(tok.nextToken().trim());
+        String entryIdDef = args[4];
+        String entryKeyDef = args[5];
+        String smlFile = args[6];
+        List attachmentFilenames = new ArrayList();
+        List attachmentNames = new ArrayList();
+        List attachmentDescriptions = new ArrayList();
+        List attachmentMimeTypes = new ArrayList();
+        for (int i = 7; i + 3 < args.length; i += 4) {
+            attachmentFilenames.add(args[i].trim());
+            attachmentNames.add(args[i+1].trim());
+            attachmentDescriptions.add(args[i+2].trim());
+            attachmentMimeTypes.add(args[i+3].trim());
+        }
         I2PAppContext ctx = I2PAppContext.getGlobalContext();
-        BlogManager mgr = new BlogManager(ctx, args[0]);
+        BlogManager mgr = new BlogManager(ctx, rootDir);
+        EntryContainer entry = createEntry(ctx, mgr, hashStr, tags, entryIdDef, entryKeyDef, smlFile, true,
+                                           attachmentFilenames, attachmentNames, attachmentDescriptions, 
+                                           attachmentMimeTypes);
+        if (entry != null)
+            mgr.getArchive().regenerateIndex();
+    }
+    
+    /**
+     * Create a new entry, storing it into the blogManager's archive and incrementing the 
+     * blog's "most recent id" setting.  This does not however regenerate the manager's index.
+     *
+     * @param blogHashStr base64(SHA256(blog public key))
+     * @param tags list of tags/categories to post under (String elements
+     * @param entryIdDef NEXT (for next entry id for the blog, or midnight of the current day),
+     *                   NOW (current time), or an explicit entry id
+     * @param entryKeyDef session key under which the entry should be encrypted
+     * @param smlFilename file in which the sml entry is to be found
+     * @param storeLocal if true, should this entry be stored in the mgr.getArchive()
+     * @param attachmentFilenames list of filenames for attachments to load
+     * @param attachmentNames list of names to use for the given attachments
+     * @param attachmentDescriptions list of descriptions for the given attachments
+     * @param attachmentMimeTypes list of mime types to use for the given attachments
+     * @return blog URI posted, or null
+     */
+    public static EntryContainer createEntry(I2PAppContext ctx, BlogManager mgr, String blogHashStr, List tags, 
+                                             String entryIdDef, String entryKeyDef, String smlFilename, boolean storeLocal,
+                                             List attachmentFilenames, List attachmentNames, 
+                                             List attachmentDescriptions, List attachmentMimeTypes) {
+        Hash blogHash = new Hash(Base64.decode(blogHashStr));
+        User user = mgr.getUser(blogHash);
         long entryId = -1;
-        if ("NOW".equals(args[4])) {
+        if ("NOW".equalsIgnoreCase(entryIdDef)) {
             entryId = ctx.clock().now();
+        } else if ("NEXT".equalsIgnoreCase(entryIdDef) || (entryIdDef == null)) {
+            entryId = mgr.getNextBlogEntry(user);
         } else {
             try {
-                entryId = Long.parseLong(args[4]);
+                entryId = Long.parseLong(entryIdDef);
             } catch (NumberFormatException nfe) {
                 nfe.printStackTrace();
-                return;
+                return null;
             }
         }
-        StringTokenizer tok = new StringTokenizer(args[3], ",");
-        String tags[] = new String[tok.countTokens()];
-        for (int i = 0; i < tags.length; i++)
-            tags[i] = tok.nextToken();
-        BlogURI uri = new BlogURI(new Hash(Base64.decode(args[2])), entryId);
+        String tagVals[] = new String[(tags != null ? tags.size() : 0)];
+        if (tags != null)
+            for (int i = 0; i < tags.size(); i++)
+                tagVals[i] = ((String)tags.get(i)).trim();
+        BlogURI uri = new BlogURI(blogHash, entryId);
         BlogInfo blog = mgr.getArchive().getBlogInfo(uri);
         if (blog == null) {
             System.err.println("Blog does not exist: " + uri);
-            return;
+            return null;
         }
         SigningPrivateKey key = mgr.getMyPrivateKey(blog);
         
         try {
-            byte smlData[] = read(args[6]);
-            EntryContainer c = new EntryContainer(uri, tags, smlData);
-            for (int i = 7; i < args.length; i++) {
-                c.addAttachment(read(args[i]), new File(args[i]).getName(), 
-                                "Attached file", "application/octet-stream");
+            byte smlData[] = read(smlFilename);
+            EntryContainer c = new EntryContainer(uri, tagVals, smlData);
+            if ( (attachmentFilenames != null) && 
+                 (attachmentFilenames.size() == attachmentNames.size()) && 
+                 (attachmentFilenames.size() == attachmentDescriptions.size()) && 
+                 (attachmentFilenames.size() == attachmentMimeTypes.size()) ) {
+                for (int i = 0; i < attachmentFilenames.size(); i++) {
+                    File attachmentFile = new File((String)attachmentFilenames.get(i));
+                    String name = (String)attachmentNames.get(i);
+                    String descr = (String)attachmentDescriptions.get(i);
+                    String mimetype = (String)attachmentMimeTypes.get(i);
+                    c.addAttachment(read(attachmentFile.getAbsolutePath()), name, descr, mimetype);
+                }
             }
             SessionKey entryKey = null;
-            if (!"NONE".equals(args[5]))
-                entryKey = new SessionKey(Base64.decode(args[5]));
+            if ( (entryKeyDef != null) && (entryKeyDef.trim().length() > 0) && (!"NONE".equalsIgnoreCase(entryKeyDef)) )
+                entryKey = new SessionKey(Base64.decode(entryKeyDef));
             c.seal(ctx, key, entryKey);
-            boolean ok = mgr.getArchive().storeEntry(c);
-            System.out.println("Blog entry created: " + c+ "? " + ok);
-            if (ok)
-                mgr.getArchive().regenerateIndex();
+            if (storeLocal) {
+                boolean ok = mgr.getArchive().storeEntry(c);
+                //System.out.println("Blog entry created: " + c+ "? " + ok);
+                if (!ok) {
+                    System.err.println("Error: store failed");
+                    return null;
+                }
+            }
+            user.setMostRecentEntry(uri.getEntryId());
+            mgr.storeUser(user); // saves even if !user.getAuthenticated()
+            return c;
         } catch (IOException ioe) {
             ioe.printStackTrace();
+            return null;
         }
     }
     

apps/syndie/java/src/net/i2p/syndie/CLIPost.java

@@ -0,0 +1,208 @@
+package net.i2p.syndie;
+
+import java.io.*;
+import java.util.*;
+import net.i2p.*;
+import net.i2p.data.*;
+import net.i2p.syndie.data.*;
+import net.i2p.util.EepPost;
+
+/**
+ * Simple CLI to post an entry.
+ * 
+ */
+public class CLIPost {
+    public static final String USAGE = "Usage: \"" + CLIPost.class.getName() + " [args]\", where args are:"
+        + "\n  --syndieDir $syndieRootDir  // syndie root dir, under which syndie.config exists"
+        + "\n  --blog $blogHash            // base64 of the blog's key"
+        + "\n  --sml $smlFile              // file with the SML entry"
+        + "\n  [--importurl ($url|none)]   // defaults to http://localhost:7657/syndie/import.jsp"
+        + "\n  [--proxyhost $hostname]     // HTTP proxy host for sending the data to the import URL"
+        + "\n  [--proxyport $portnum]      // HTTP proxy port for sending the data to the import URL"
+        + "\n  [--storelocal (true|false)] // should it be stored directly with the file system"
+        + "\n                              // (false by default, since its stored locally via importurl)"
+        + "\n  [--entryId ($num|next|now)] // entryId to use: explicit, the blog's next (default), or timestamp"
+        + "\n  [--attachment$N $file $name $desc $type]"
+        + "\n                              // Nth file / suggested name / description / mime type";
+    
+    public static void main(String args[]) {
+        String rootDir = getArg(args, "syndieDir");
+        String hashStr = getArg(args, "blog");
+        String smlFile = getArg(args, "sml");
+        if ( (rootDir == null) || (hashStr == null) || (smlFile == null) ) {
+            System.err.println(USAGE);
+            return;
+        }
+        
+        String url = getArg(args, "importurl");
+        String entryIdDef = getArg(args, "entryId");
+        
+        List attachmentFilenames = new ArrayList();
+        List attachmentNames = new ArrayList();
+        List attachmentDescriptions = new ArrayList();
+        List attachmentMimeTypes = new ArrayList();
+        while (true) {
+            // --attachment$N $file $name $desc $type]
+            String file = getAttachmentParam(args, attachmentFilenames.size(), 0);
+            String name = getAttachmentParam(args, attachmentFilenames.size(), 1);
+            String desc = getAttachmentParam(args, attachmentFilenames.size(), 2);
+            String type = getAttachmentParam(args, attachmentFilenames.size(), 3);
+            if ( (file != null) && (name != null) && (desc != null) && (type != null) ) {
+                attachmentFilenames.add(file);
+                attachmentNames.add(name);
+                attachmentDescriptions.add(desc);
+                attachmentMimeTypes.add(type);
+            } else {
+                break;
+            }
+        }
+        
+        List tags = readTags(smlFile);
+        
+        // don't support the entry key stuff yet...
+        String entryKeyDef = null; //args[5];
+        
+        String loc = getArg(args, "storelocal");
+        boolean storeLocal = false;
+        if (loc != null)
+            storeLocal = Boolean.valueOf(loc).booleanValue();
+        
+        if (!storeLocal && "none".equalsIgnoreCase(url)) {
+            System.err.println("You need to post it somewhere, so either specify \"--storelocal true\"");
+            System.err.println("or don't specify \"--importurl none\"");
+            return;
+        }
+        
+        I2PAppContext ctx = I2PAppContext.getGlobalContext();
+        BlogManager mgr = new BlogManager(ctx, rootDir, false);
+        EntryContainer entry = CLI.createEntry(ctx, mgr, hashStr, tags, entryIdDef, entryKeyDef, smlFile, storeLocal, 
+                                               attachmentFilenames, attachmentNames, attachmentDescriptions, 
+                                               attachmentMimeTypes);
+        if (entry != null) {
+            if (storeLocal)
+                mgr.getArchive().regenerateIndex();
+            if (!("none".equalsIgnoreCase(url))) {
+                if ( (url == null) || (url.trim().length() <= 0) )
+                    url = "http://localhost:7657/syndie/import.jsp";
+
+                // now send it to the import URL
+                BlogInfo info = mgr.getArchive().getBlogInfo(entry.getURI().getKeyHash());
+                File fMeta = null;
+                File fData = null;
+
+                try {
+                    fMeta = File.createTempFile("cli", ".snm", mgr.getTempDir());
+                    fData = File.createTempFile("cli", ".snd", mgr.getTempDir());
+                    FileOutputStream out = new FileOutputStream(fMeta);
+                    info.write(out);
+                    out.close();
+                    out = new FileOutputStream(fData);
+                    entry.write(out, true);
+                    out.close();
+                    fMeta.deleteOnExit();
+                    fData.deleteOnExit();
+                } catch (IOException ioe) {
+                    System.err.println("Error writing temp files: " + ioe.getMessage());
+                    return;
+                }
+
+                Map uploads = new HashMap(2);
+                uploads.put("blogmeta0", fMeta);
+                uploads.put("blogpost0", fData);
+
+                String proxyHost = getArg(args, "proxyhost");
+                String proxyPortStr = getArg(args, "proxyport");
+                int proxyPort = -1;
+                if (proxyPortStr != null) 
+                    try { proxyPort = Integer.parseInt(proxyPortStr); } catch (NumberFormatException nfe) { }
+
+                OnCompletion job = new OnCompletion();
+                EepPost post = new EepPost();
+                post.postFiles(url, (proxyPort > 0 ? proxyHost : null), proxyPort, uploads, job);
+                boolean posted = job.waitForCompletion(30*1000);
+                if (posted)
+                    System.out.println("Posted successfully: " + entry.getURI().toString());
+                else
+                    System.out.println("Posting failed");
+            } else if (storeLocal) {
+                System.out.println("Store local successfully: " + entry.getURI().toString());
+            } else {
+                // foo
+            }
+        } else {
+            System.err.println("Error creating the blog entry");
+        }
+    }
+    
+    private static class OnCompletion implements Runnable {
+        private boolean _complete;
+        public OnCompletion() { _complete = false; }
+        public void run() { 
+            _complete = true; 
+            synchronized (OnCompletion.this) { 
+                OnCompletion.this.notifyAll();
+            }
+        }
+        public boolean waitForCompletion(long max) {
+            long end = max + System.currentTimeMillis();
+            while (!_complete) {
+                long now = System.currentTimeMillis();
+                if (now >= end)
+                    return false;
+                try {
+                    synchronized (OnCompletion.this) {
+                        OnCompletion.this.wait(end-now);
+                    }
+                } catch (InterruptedException ie) {}
+            }
+            return true;
+        }
+    }
+
+    private static String getArg(String args[], String param) {
+        if (args != null) 
+            for (int i = 0; i + 1< args.length; i++)
+                if (args[i].equalsIgnoreCase("--"+param))
+                    return args[i+1];
+        return null;
+    }
+    private static String getAttachmentParam(String args[], int attachmentNum, int paramNum) {
+        if (args != null) 
+            for (int i = 0; i + 4 < args.length; i++)
+                if (args[i].equalsIgnoreCase("--attachment"+attachmentNum))
+                    return args[i+1+paramNum];
+        return null;
+    }
+    
+    private static List readTags(String smlFile) {
+        BufferedReader in = null;
+        try {
+            in = new BufferedReader(new InputStreamReader(new FileInputStream(smlFile), "UTF-8"));
+            String line = null;
+            while ( (line = in.readLine()) != null) {
+                if (line.length() <= 0)
+                    return new ArrayList();
+                else if (line.startsWith("Tags:"))
+                    return parseTags(line.substring("Tags:".length()));
+            }
+            return null;
+        } catch (IOException ioe) {
+            return new ArrayList();
+        } finally {
+            if (in != null) try { in.close(); } catch (IOException ioe) {}
+        }
+    }
+    
+    private static List parseTags(String tags) {
+        if (tags == null) 
+            return new ArrayList();
+        StringTokenizer tok = new StringTokenizer(tags, " ,\t\n");
+        List rv = new ArrayList();
+        while (tok.hasMoreTokens()) {
+            String cur = tok.nextToken().trim();
+            if (cur.length() > 0)
+                rv.add(cur);
+        }
+        return rv;
+    }
+}

apps/syndie/java/src/net/i2p/syndie/Sucker.java

@@ -0,0 +1,635 @@
+package net.i2p.syndie;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import sun.security.provider.SHA;
+
+import com.sun.syndication.feed.atom.Entry;
+import com.sun.syndication.feed.synd.SyndCategory;
+import com.sun.syndication.feed.synd.SyndContent;
+import com.sun.syndication.feed.synd.SyndEntry;
+import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.io.FeedException;
+import com.sun.syndication.io.SyndFeedInput;
+import com.sun.syndication.io.XmlReader;
+
+import net.i2p.I2PAppContext;
+import net.i2p.util.EepGet;
+
+public class Sucker {
+    private String urlToLoad;
+    private String outputDir="./sucker_out";
+    private String historyPath="./sucker.history";
+    private String feedTag="feed";
+    private File historyFile;
+    private String proxyPort;
+    private String proxyHost;
+    private String pushScript;
+    private int attachmentCounter=0;
+    private String messagePath;
+    private String baseUrl;
+    private boolean importEnclosures=true;
+    private boolean importRefs=true;
+    private boolean pendingEndLink;
+    
+    public Sucker() {}
+    
+    public boolean parseArgs(String args[]) {
+        for (int i = 0; i < args.length; i++) {
+            if ("--load".equals(args[i]))
+                urlToLoad = args[++i];
+            if ("--outputdir".equals(args[i]))
+                outputDir = args[++i];
+            if ("--history".equals(args[i]))
+                historyPath = args[++i];
+            if ("--tag".equals(args[i]))
+                feedTag = args[++i];
+            if ("--proxyhost".equals(args[i]))
+                proxyHost = args[++i];
+            if ("--proxyport".equals(args[i]))
+                proxyPort = args[++i];
+            if ("--exec".equals(args[i]))
+                pushScript = args[++i];
+            if ("--importenclosures".equals(args[i]))
+                importEnclosures= args[++i].equals("true");
+            if ("--importenrefs".equals(args[i]))
+                importRefs= args[++i].equals("true");
+        }
+
+        // Cut ending '/' from outputDir
+        if (outputDir.endsWith("/"))
+            outputDir = outputDir.substring(0, outputDir.length() - 1);
+
+        if (urlToLoad == null)
+            return false;
+        
+        // Find base url, gah HELP
+        int idx=urlToLoad.length();
+        int x=urlToLoad.indexOf('?');
+        if(x>0)
+            idx=x;
+        while(idx>0)
+        {
+            idx--;
+            if(urlToLoad.charAt(idx)=='/')
+                break;
+        }
+        if(idx==0)
+            idx=x;
+        baseUrl=urlToLoad.substring(0,idx);
+        System.out.println("BaseUrl: "+baseUrl);
+
+        return true;
+    }
+    
+    public void suck() {
+        URL feedUrl;
+        SyndFeed feed;
+        int i;
+
+        //if (proxyHost != null && proxyPort != null) {
+        //    // Set proxy
+        //    System.setProperty("http.proxyHost", proxyHost);
+        //    System.setProperty("http.proxyPort", proxyPort);
+        //}
+
+
+        //
+        try {
+            
+            // Create outputDir if missing
+            File f = new File(outputDir);
+            f.mkdirs();
+
+            // Create historyFile if missing
+            historyFile = new File(historyPath);
+            if (!historyFile.exists())
+                historyFile.createNewFile();
+
+            int messageNumber;
+
+            File lastIdFile = new File(historyPath + ".lastId");
+            if (!lastIdFile.exists())
+                lastIdFile.createNewFile();
+
+            FileInputStream fis = new FileInputStream(lastIdFile);
+            String number = readLine(fis);
+            try {
+                messageNumber = Integer.parseInt(number);
+            } catch (NumberFormatException e) {
+                messageNumber = 0;
+            }
+
+            SyndFeedInput input = new SyndFeedInput();
+
+            boolean shouldProxy = false;
+            int proxyPortNum = -1;
+            if ( (proxyHost != null) && (proxyPort != null) ) {
+                try {
+                    proxyPortNum = Integer.parseInt(proxyPort);
+                    if (proxyPortNum > 0)
+                        shouldProxy = true;
+                } catch (NumberFormatException nfe) {
+                    nfe.printStackTrace();
+                }
+            }
+            int numRetries = 2;
+            // perhaps specify a temp dir?
+            File fetched = File.createTempFile("sucker", ".fetch");
+            fetched.deleteOnExit();
+            // we use eepGet, since it retries and doesn't leak DNS requests like URL does
+            EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum, 
+                                    numRetries, fetched.getAbsolutePath(), urlToLoad);
+            SuckerFetchListener lsnr = new SuckerFetchListener();
+            get.addStatusListener(lsnr);
+            get.fetch();
+            boolean ok = lsnr.waitForSuccess();
+            if (!ok) {
+                System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
+                return;
+            }
+            
+            feed = input.build(new XmlReader(fetched));
+
+            List entries = feed.getEntries();
+
+            FileOutputStream hos = new FileOutputStream(historyFile, true);
+
+            ListIterator iter = entries.listIterator();
+            while (iter.hasNext()) {
+                
+                attachmentCounter=0;
+                
+                SyndEntry e = (SyndEntry) iter.next();
+                // Calculate messageId
+                String feedHash = sha1(urlToLoad);
+                String itemHash = sha1(e.getTitle() + e.getDescription());
+                Date d = e.getPublishedDate();
+                String time;
+                if(d!=null)
+                    time = "" + d.getTime();
+                else
+                    time = "" + new Date().getTime();
+                    
+                String outputFileName = outputDir + "/" + messageNumber;
+
+                /*
+                 * $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
+                 * hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
+                 * hash of the SyndEntry, $time would be the time that the entry
+                 * was posted $outputfile would be the $outputdir/$uniqueid
+                 */
+                String messageId = feedHash + ":" + itemHash + ":" + time + ":"
+                                   + outputFileName;
+
+                // Check if we already have this
+                if (!existsInHistory(messageId)) {
+                    System.out.println("new: " + messageId);
+
+                    if (convertToSml(e, ""+messageNumber)) {
+                        hos.write(messageId.getBytes());
+                        hos.write("\n".getBytes());
+
+                        if (pushScript != null) {
+                            if (!execPushScript(""+messageNumber, time))
+                                System.out.println("push failed");
+                            else
+                                System.out.println("push success");
+                        }
+                    }
+                    messageNumber++;
+                }
+            }
+
+            FileOutputStream fos = new FileOutputStream(lastIdFile);
+            fos.write(("" + messageNumber).getBytes());
+        } catch (MalformedURLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (IllegalArgumentException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (FeedException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        System.out.println("Done.");
+    }
+
+    public static void main(String[] args) {
+        Sucker sucker = new Sucker();
+        boolean ok = sucker.parseArgs(args);
+        if (!ok) {
+            usage();
+            return;
+        }
+        
+        sucker.suck();
+    }
+
+    private boolean execPushScript(String id, String time) {
+        try {
+            String ls_str;
+
+            String cli = pushScript + " " + outputDir + " " + id + " " + time;
+            Process pushScript_proc = Runtime.getRuntime().exec(cli);
+
+            // get its output (your input) stream
+
+            DataInputStream ls_in = new DataInputStream(pushScript_proc.getInputStream());
+
+            try {
+                while ((ls_str = ls_in.readLine()) != null) {
+                    System.out.println(pushScript + ": " + ls_str);
+                }
+            } catch (IOException e) {
+                return false;
+            }
+            try {
+                pushScript_proc.waitFor();
+                if(pushScript_proc.exitValue()==0)
+                    return true;
+            } catch (InterruptedException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+            return false;
+        } catch (IOException e1) {
+            System.err.println(e1);
+            return false;
+        }
+    }
+
+    private boolean convertToSml(SyndEntry e, String messageName) {
+
+        // Create message
+        File messageFile = new File(messageName);
+        FileOutputStream fos;
+        messagePath=outputDir+"/"+messageName;
+        try {
+            fos = new FileOutputStream(messagePath);
+
+            String sml;
+            sml = "Subject: " + e.getTitle() + "\n";
+            List cats = e.getCategories();
+            Iterator iter = cats.iterator();
+            String tags = feedTag;
+            while (iter.hasNext()) {
+                SyndCategory c = (SyndCategory) iter.next();
+                String tag=c.getName();
+                tag=tag.replaceAll("[^a-zA-z.-_:]","_");
+                tags += "\t" + feedTag + "." + tag;
+            }
+            sml += "Tags: " + tags + "\n";
+            sml += "\n";
+
+            SyndContent content;
+
+            List l = e.getContents();
+            if(l!=null)
+            {
+            System.out.println("There is content");
+                iter = l.iterator();
+                while(iter.hasNext())
+                {
+                    content = (SyndContent)iter.next();
+                    String c = content.getValue();
+                    System.out.println("Content: "+c);
+                                  sml += htmlToSml(c);
+                                  sml += "\n";
+                }
+            }
+            String source=e.getUri();
+            if(source.indexOf("http")<0)
+            source=baseUrl+source;
+            sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
+
+            fos.write(sml.getBytes());
+
+            return true;
+        } catch (FileNotFoundException e1) {
+            // TODO Auto-generated catch block
+            e1.printStackTrace();
+        } catch (IOException e2) {
+            // TODO Auto-generated catch block
+            e2.printStackTrace();
+        }
+        return false;
+    }
+
+    private String htmlToSml(String html) {
+
+        String sml="";
+        int i;
+
+        pendingEndLink=false;
+
+        for(i=0;i<html.length();)
+        {
+            if(html.charAt(i)=='<')
+            {
+                //System.out.println("html: "+html.substring(i));
+                
+                int tagLen = findTagLen(html.substring(i));
+                //
+                String htmlTag = html.substring(i,i+tagLen);
+                
+                //System.out.println("htmlTag: "+htmlTag);
+                
+                String smlTag = htmlTagToSmlTag(htmlTag);
+                if(smlTag!=null)
+                    sml+=smlTag;
+                i+=tagLen;
+                //System.out.println("tagLen: "+tagLen);
+                sml+=" "; 
+            }
+            else
+            {
+                sml+=html.charAt(i++);
+            }
+        }
+        
+        return sml;
+    }
+
+    private String htmlTagToSmlTag(String htmlTag) {
+        String ret="";
+
+        if(importEnclosures && htmlTag.startsWith("<img"))
+        {
+            System.out.println("Found image tag: "+htmlTag);
+            int a,b;
+            a=htmlTag.indexOf("src=\"")+5;
+            b=a+1;
+            while(htmlTag.charAt(b)!='\"')
+                b++;
+            String imageLink=htmlTag.substring(a,b);
+            
+            if(pendingEndLink) {
+                ret="[/link]";
+            pendingEndLink=false;
+            }
+    
+            ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
+            
+            a=htmlTag.indexOf("alt=\"")+5;
+            if(a>=5)
+            {
+                b=a+1;
+                while(htmlTag.charAt(b)!='\"')
+                    b++;
+                String altText=htmlTag.substring(a,b);
+                ret+=altText;
+            }
+            
+            ret+="[/img]";
+            
+            if(imageLink.indexOf("http")<0)
+                imageLink=baseUrl+"/"+imageLink;
+            
+            fetchAttachment(imageLink);
+
+            System.out.println("Converted to: "+ret);
+            
+            return ret;
+            
+        }
+        if(importRefs && htmlTag.startsWith("<a "))
+        {
+            System.out.println("Found link tag: "+htmlTag);
+            int a,b;
+            
+            a=htmlTag.indexOf("href=\"")+6;
+            b=a+1;
+            while(htmlTag.charAt(b)!='\"')
+                b++;
+            String link=htmlTag.substring(a,b);
+            if(link.indexOf("http")<0)
+                link=baseUrl+"/"+link;
+            
+            String schema="web";
+            
+            ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
+            pendingEndLink=true;
+        
+            System.out.println("Converted to: "+ret);
+
+            return ret;
+        }
+        
+        if ("</a>".equals(htmlTag)) {
+            if (pendingEndLink)
+                return "[/link]";
+        }
+        
+        if("<b>".equals(htmlTag))
+            return "[b]";
+        if("</b>".equals(htmlTag))
+            return "[/b]";
+        if("<i>".equals(htmlTag))
+            return "[i]";
+        if("</i>".equals(htmlTag))
+            return "[/i]";
+        
+        return null;
+    }
+
+    private void fetchAttachment(String link) {
+        System.out.println("Fetch attachment from: "+link);
+        String attachmentPath = messagePath+"."+attachmentCounter;
+        try {
+            link=link.replaceAll("&amp;","&");
+            URL attachmentUrl = new URL(link);
+            InputStream is = attachmentUrl.openStream();
+            
+            FileOutputStream fos = new FileOutputStream(attachmentPath);
+            
+            while(true)
+            {
+                int i =is.read();
+                if(i<0)
+                    break;
+                fos.write(i);
+            }
+            
+        } catch (MalformedURLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        
+        attachmentCounter++;
+    }
+
+    private static int findTagLen(String s) {
+        int i;
+        for(i=0;i<s.length();i++)
+        {
+            if(s.charAt(i)=='>')
+                return i+1;
+            if(s.charAt(i)=='"')
+            {
+                i++;
+                while(s.charAt(i)!='"')
+                    i++;
+            }   
+        }
+        System.out.println("WTF");
+        return 0;
+    }
+
+    private boolean existsInHistory(String messageId) {
+        int idx;
+        idx = messageId.lastIndexOf(":");
+        String lineToCompare = messageId.substring(0, idx-1);
+        idx = lineToCompare.lastIndexOf(":");
+        lineToCompare = lineToCompare.substring(0, idx-1);
+        try {
+            FileInputStream his = new FileInputStream(historyFile);
+            String line;
+            while ((line = readLine(his)) != null) {
+                idx = line.lastIndexOf(":");
+                if (idx < 0)
+                    return false;
+                line = line.substring(0, idx-1);
+                idx = line.lastIndexOf(":");
+                if (idx < 0)
+                    return false;
+                line = line.substring(0, idx-1);
+                if (line.equals(lineToCompare))
+                    return true;
+            }
+        } catch (FileNotFoundException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        return false;
+    }
+
+    private static void usage() {
+        System.out.println("sucker --load $urlToFeed \n"
+                + "--proxyhost <host> \n" 
+                + "--proxyport <port> \n"
+                + "--importenclosures true \n" 
+                + "--importrefs true \n"
+                + "--tag feed \n" 
+                + "--outputdir ./sucker_out \n"
+                + "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+                + "--history ./sucker.history");
+        System.exit(1);
+    }
+
+    private static String sha1(String s) {
+        try {
+            MessageDigest md = MessageDigest.getInstance("SHA");
+            md.update(s.getBytes());
+            byte[] buf = md.digest();
+            String ret = new sun.misc.BASE64Encoder().encode(buf);
+            return ret;
+        } catch (NoSuchAlgorithmException ex) {
+            ex.printStackTrace();
+        }
+        return null;
+    }
+
+    private static String readLine(FileInputStream in) {
+        StringBuffer sb = new StringBuffer();
+        int c = 0;
+        while (true) {
+            try {
+                c = in.read();
+            } catch (IOException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+                break;
+            }
+            if (c < 0)
+                break;
+            if (c == '\n')
+                break;
+            sb.append((char) c);
+        }
+        // TODO Auto-generated method stub
+        return sb.toString();
+    }
+}
+
+/**
+ * Simple blocking listener for eepget.  block in waitForSuccess().
+ */
+class SuckerFetchListener implements EepGet.StatusListener {
+    private volatile boolean _complete;
+    private volatile boolean _successful;
+    
+    public SuckerFetchListener() {
+        _complete = false;
+        _successful = false;
+    }
+    
+    public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {
+        notifyComplete(true);
+    }
+    
+    public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {
+        notifyComplete(false);
+    }
+    
+    private void notifyComplete(boolean ok) {
+        synchronized (this) {
+            _complete = true;
+            _successful = ok;
+            notifyAll();
+        }
+    }
+    
+    /**
+     * Block until the fetch is successful, returning true if it did fetch completely, 
+     * false if it didn't.
+     *
+     */
+    public boolean waitForSuccess() {
+        while (!_complete) {
+            try {
+                synchronized (this) {
+                    wait();
+                }
+            } catch (InterruptedException ie) {}
+        }
+        return _successful;
+    }
+    
+    public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
+        // noop, it may retry
+    }
+    public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {
+        // ignore this status update
+    }
+    public void headerReceived(String url, int currentAttempt, String key, String val) {
+        // ignore
+    }
+    
+}

apps/syndie/java/src/net/i2p/syndie/User.java

@@ -16,6 +16,7 @@ public class User {
     private String _username;
     private String _hashedPassword;
     private Hash _blog;
+    private String _userHash;
     private long _mostRecentEntry;
     /** Group name to List of blog selectors, where the selectors are of the form
      * blog://$key, entry://$key/$entryId, blogtag://$key/$tag, tag://$tag
@@ -39,6 +40,9 @@ public class User {
     private String _torProxyHost;
     private int _torProxyPort;
     private PetNameDB _petnames;
+    private boolean _importAddresses;
+
+    static final String PROP_USERHASH = "__userHash";
     
     public User() {
         _context = I2PAppContext.getGlobalContext();
@@ -47,6 +51,7 @@ public class User {
     private void init() {
         _authenticated = false;
         _username = null;
+        _userHash = null;
         _hashedPassword = null;
         _blog = null;
         _mostRecentEntry = -1;
@@ -66,10 +71,12 @@ public class User {
         _lastLogin = -1;
         _lastMetaEntry = 0;
         _petnames = new PetNameDB();
+        _importAddresses = false;
     }
     
     public boolean getAuthenticated() { return _authenticated; }
     public String getUsername() { return _username; }
+    public String getUserHash() { return _userHash; }
     public Hash getBlog() { return _blog; }
     public String getBlogStr() { return Base64.encode(_blog.getData()); }
     public long getMostRecentEntry() { return _mostRecentEntry; }
@@ -97,6 +104,7 @@ public class User {
     public int getTorProxyPort() { return _torProxyPort; }
     
     public PetNameDB getPetNameDB() { return _petnames; }
+    public boolean getImportAddresses() { return _importAddresses; }
     
     public void invalidate() { 
         if (_authenticated)
@@ -105,15 +113,22 @@ public class User {
     }
     
     public String login(String login, String pass, Properties props) {
-        String expectedPass = props.getProperty("password");
+        _username = login;
+        load(props);
         String hpass = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(pass)).getData());
-        if (!hpass.equals(expectedPass)) {
-            _authenticated = false;
+        if (!hpass.equals(_hashedPassword)) {
             return "<span class=\"b_loginMsgErr\">Incorrect password</span>";
         }
-        
-        _username = login;
-        _hashedPassword = expectedPass;
+        _lastLogin = _context.clock().now();
+        _authenticated = true;
+        return LOGIN_OK;
+    }
+    
+    
+    public void load(Properties props) {
+        _authenticated = false;
+        _hashedPassword = props.getProperty("password");
+        _userHash = props.getProperty(PROP_USERHASH);
         
         // blog=luS9d3uaf....HwAE=
         String b = props.getProperty("blog");
@@ -185,9 +200,8 @@ public class User {
         _eepProxyHost = props.getProperty("eepproxyhost");
         _webProxyHost = props.getProperty("webproxyhost");
         _torProxyHost = props.getProperty("torproxyhost");
-        _lastLogin = _context.clock().now();
-        _authenticated = true;
-        return LOGIN_OK;
+        String importadr = props.getProperty("importaddresses", "false");
+        _importAddresses = (importadr != null) && (importadr.equals("true"));
     }
     
     private int getInt(String val) {
@@ -209,7 +223,7 @@ public class User {
         buf.append("showexpanded=" + getShowExpanded() + "\n");
         buf.append("defaultselector=" + getDefaultSelector() + "\n");
         buf.append("allowaccessremote=" + _allowAccessRemote + "\n");
-        
+        buf.append("importaddresses=" + getImportAddresses() + "\n");
         buf.append("groups=");
         Map groups = getBlogGroups();
         for (Iterator iter = groups.keySet().iterator(); iter.hasNext(); ) {

apps/syndie/java/src/net/i2p/syndie/sml/HTMLPreviewRenderer.java

@@ -4,6 +4,7 @@ import java.io.*;
 import java.text.*;
 import java.util.*;
 import net.i2p.I2PAppContext;
+import net.i2p.client.naming.PetName;
 import net.i2p.data.*;
 import net.i2p.syndie.*;
 import net.i2p.syndie.data.*;
@@ -101,13 +102,13 @@ public class HTMLPreviewRenderer extends HTMLRenderer {
             _postBodyBuffer.append(getSpan("summDetailAddr")).append("Addresses:</span>");
             for (int i = 0; i < _addresses.size(); i++) {
                 Address a = (Address)_addresses.get(i);
-
-                String knownName = null;
+                importAddress(a);
+                PetName pn = null;
                 if (_user != null)
-                    knownName = _user.getPetNameDB().getNameByLocation(a.location);
-                if (knownName != null) {
+                    pn = _user.getPetNameDB().getByLocation(a.location);
+                if (pn != null) {
                     _postBodyBuffer.append(' ').append(getSpan("summDetailAddrKnown"));
-                    _postBodyBuffer.append(sanitizeString(knownName)).append("</span>");
+                    _postBodyBuffer.append(sanitizeString(pn.getName())).append("</span>");
                 } else {
                     _postBodyBuffer.append(" <a ").append(getClass("summDetailAddrLink")).append(" href=\"addresses.jsp?");
                     if (a.schema != null)
@@ -119,7 +120,7 @@ public class HTMLPreviewRenderer extends HTMLRenderer {
                     if (a.protocol != null)
                         _postBodyBuffer.append("protocol=").append(sanitizeTagParam(a.protocol)).append('&');
                     _postBodyBuffer.append("\">").append(sanitizeString(a.name)).append("</a>");
-                }
+                } 
             }
             _postBodyBuffer.append("<br />\n");
         }
@@ -132,7 +133,7 @@ public class HTMLPreviewRenderer extends HTMLRenderer {
                 _postBodyBuffer.append("\">").append(sanitizeString(a.name)).append("</a>");
                 if (a.description != null)
                     _postBodyBuffer.append(": ").append(getSpan("summDetailArchiveDesc")).append(sanitizeString(a.description)).append("</span>");
-                if (null == _user.getPetNameDB().getNameByLocation(a.location)) {
+                if (null == _user.getPetNameDB().getByLocation(a.location)) {
                     _postBodyBuffer.append(" <a ").append(getClass("summDetailArchiveBookmark")).append(" href=\"");
                     _postBodyBuffer.append(getBookmarkURL(a.name, a.location, a.locationSchema, "syndiearchive"));
                     _postBodyBuffer.append("\">bookmark</a>");

apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java

@@ -148,6 +148,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     
     public void receivePlain(String text) { 
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive plain [" + text + "]");
         _bodyBuffer.append(sanitizeString(text)); 
     }
     
@@ -165,6 +166,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     }
     public void receiveHR() {
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive HR");
         _bodyBuffer.append(getSpan("hr")).append("<hr /></span>");
     }
     public void receiveH1(String body) {
@@ -189,6 +191,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     }
     public void receivePre(String body) {
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive pre: [" + sanitizeString(body) + "]");
         _bodyBuffer.append("<pre ").append(getClass("pre")).append(" >").append(sanitizeString(body)).append("</pre>");
     }
     
@@ -245,6 +248,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     
     public void receiveNewline() { 
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive NL");
         if (true || (_lastNewlineAt >= _bodyBuffer.length()))
             _bodyBuffer.append(getSpan("nl")).append("<br /></span>\n");
         else
@@ -261,10 +265,12 @@ public class HTMLRenderer extends EventReceiverImpl {
     public void receiveBegin() {}
     public void receiveLeftBracket() { 
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive [");
         _bodyBuffer.append(getSpan("lb")).append("[</span>");
     }
     public void receiveRightBracket() { 
         if (!continueBody()) { return; }
+        if (_log.shouldLog(Log.DEBUG)) _log.debug("receive ]");
         _bodyBuffer.append(getSpan("rb")).append("]</span>");
     }
     
@@ -441,6 +447,30 @@ public class HTMLRenderer extends EventReceiverImpl {
             return DataHelper.eq(schema, a.schema) && DataHelper.eq(location, a.location) && DataHelper.eq(protocol, a.protocol) && DataHelper.eq(name, a.name);
         }
     }
+    
+    public void importAddress(Address a) {
+        if (_user != null && _user.getImportAddresses() && !_user.getPetNameDB().containsName(a.name)) {
+            PetName pn = new PetName(a.name, a.schema, a.protocol, a.location);
+            _user.getPetNameDB().add(pn);
+            try {
+                _user.getPetNameDB().store(_user.getAddressbookLocation());
+            } catch (IOException ioe) {
+                //ignore
+            }
+        }
+        if (BlogManager.instance().getImportAddresses() 
+                && I2PAppContext.getGlobalContext().namingService().lookup(a.name) == null 
+                && a.schema.equalsIgnoreCase("i2p")) {
+            PetName pn = new PetName(a.name, a.schema, a.protocol, a.location);
+            I2PAppContext.getGlobalContext().petnameDb().add(pn);
+            try {
+                I2PAppContext.getGlobalContext().petnameDb().store();
+            } catch (IOException ioe) {
+                //ignore
+            }
+        }
+    }
+    
     public void receiveAddress(String name, String schema, String protocol, String location, String anchorText) {
         Address a = new Address();
         a.name = name;
@@ -451,12 +481,12 @@ public class HTMLRenderer extends EventReceiverImpl {
             _addresses.add(a);
         if (!continueBody()) { return; }
         if ( (schema == null) || (location == null) ) return;
-        String knownName = null;
+        PetName pn = null;
         if (_user != null)
-            knownName = _user.getPetNameDB().getNameByLocation(location);
-        if (knownName != null) {
+            pn = _user.getPetNameDB().getByLocation(location);
+        if (pn != null) {
             _bodyBuffer.append(getSpan("addr")).append(sanitizeString(anchorText)).append("</span>");
-            _bodyBuffer.append(getSpan("addrKnownName")).append("(").append(sanitizeString(knownName)).append(")</span>");
+            _bodyBuffer.append(getSpan("addrKnownName")).append("(").append(sanitizeString(pn.getName())).append(")</span>");
         } else {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Receiving address [" + location + "]");
@@ -615,13 +645,13 @@ public class HTMLRenderer extends EventReceiverImpl {
                 _postBodyBuffer.append(getSpan("summDetailAddr")).append("Addresses:</span>");
                 for (int i = 0; i < _addresses.size(); i++) {
                     Address a = (Address)_addresses.get(i);
-                    
-                    String knownName = null;
+                    importAddress(a);
+                    PetName pn = null;
                     if (_user != null)
-                        knownName = _user.getPetNameDB().getNameByLocation(a.location);
-                    if (knownName != null) {
+                        pn = _user.getPetNameDB().getByLocation(a.location);
+                    if (pn != null) {
                         _postBodyBuffer.append(' ').append(getSpan("summDetailAddrKnown"));
-                        _postBodyBuffer.append(sanitizeString(knownName)).append("</span>");
+                        _postBodyBuffer.append(sanitizeString(pn.getName())).append("</span>");
                     } else {
                         _postBodyBuffer.append(" <a ").append(getClass("summDetailAddrLink")).append(" href=\"addresses.jsp?");
                         if (a.schema != null)
@@ -633,7 +663,7 @@ public class HTMLRenderer extends EventReceiverImpl {
                         if (a.protocol != null)
                             _postBodyBuffer.append("protocol=").append(sanitizeTagParam(a.protocol)).append('&');
                         _postBodyBuffer.append("\">").append(sanitizeString(a.name)).append("</a>");
-                    }
+                    }                    
                 }
                 _postBodyBuffer.append("<br />\n");
             }
@@ -646,7 +676,7 @@ public class HTMLRenderer extends EventReceiverImpl {
                     _postBodyBuffer.append("\">").append(sanitizeString(a.name)).append("</a>");
                     if (a.description != null)
                         _postBodyBuffer.append(": ").append(getSpan("summDetailArchiveDesc")).append(sanitizeString(a.description)).append("</span>");
-                    if (null == _user.getPetNameDB().getNameByLocation(a.location)) {
+                    if (null == _user.getPetNameDB().getByLocation(a.location)) {
                         _postBodyBuffer.append(" <a ").append(getClass("summDetailArchiveBookmark")).append(" href=\"");
                         _postBodyBuffer.append(getBookmarkURL(a.name, a.location, a.locationSchema, "syndiearchive"));
                         _postBodyBuffer.append("\">bookmark it</a>");
@@ -749,9 +779,9 @@ public class HTMLRenderer extends EventReceiverImpl {
         _preBodyBuffer.append("<td nowrap=\"nowrap\" align=\"right\" valign=\"top\" ");
         _preBodyBuffer.append(getClass("meta")).append(">\n");
         
-        String knownName = null;
+        PetName pn = null;
         if ( (_entry != null) && (_user != null) )
-            knownName = _user.getPetNameDB().getNameByLocation(_entry.getURI().getKeyHash().toBase64());
+            pn = _user.getPetNameDB().getByLocation(_entry.getURI().getKeyHash().toBase64());
         //if (knownName != null)
         //    _preBodyBuffer.append("Pet name: ").append(sanitizeString(knownName)).append(" ");
 
@@ -760,8 +790,8 @@ public class HTMLRenderer extends EventReceiverImpl {
             info = _archive.getBlogInfo(_entry.getURI());
         if (info != null) {
             _preBodyBuffer.append("<a ").append(getClass("metaLink")).append(" href=\"").append(getMetadataURL()).append("\">");
-            if (knownName != null) {
-                _preBodyBuffer.append(getSpan("metaKnown")).append(sanitizeString(knownName)).append("</span>");
+            if (pn != null) {
+                _preBodyBuffer.append(getSpan("metaKnown")).append(sanitizeString(pn.getName())).append("</span>");
             } else {
                 String nameStr = info.getProperty("Name");
                 if (nameStr == null)
@@ -776,7 +806,6 @@ public class HTMLRenderer extends EventReceiverImpl {
 
         
         if ( (_user != null) && (_user.getAuthenticated()) && (_entry != null) ) {
-            PetName pn = _user.getPetNameDB().get(knownName);
             if ( (pn == null) || (!pn.isMember("Favorites")) )
                 _preBodyBuffer.append(" <input ").append(getClass("bookmark")).append(" type=\"submit\" name=\"action\" value=\"Bookmark blog\" />");
             if ( (pn == null) || (!pn.isMember("Ignore")) )
@@ -863,12 +892,17 @@ public class HTMLRenderer extends EventReceiverImpl {
         }
         if (!unsafe) return str;
         
-        str = str.replace('<', '_'); // this should be &lt;
-        str = str.replace('>', '-'); // this should be &gt;
+        //str = str.replace('<', '_'); // this should be &lt;
+        //str = str.replace('>', '-'); // this should be &gt;
+        str = str.replaceAll("<", "&lt;");
+        str = str.replaceAll(">", "&gt;");
         if (!allowNL) {
-            str = str.replace('\n', ' ');
-            str = str.replace('\r', ' ');
-            str = str.replace('\f', ' ');
+            //str = str.replace('\n', ' ');
+            //str = str.replace('\r', ' ');
+            //str = str.replace('\f', ' ');
+            str = str.replaceAll("\n", "<br />"); // no class
+            str = str.replaceAll("\r", "<br />"); // no class
+            str = str.replaceAll("\f", "<br />"); // no class
         }
         return str;
     }
@@ -879,7 +913,8 @@ public class HTMLRenderer extends EventReceiverImpl {
     }
     public static final String sanitizeTagParam(String str) {
         if (str == null) return "";
-        str = str.replace('&', '_'); // this should be &amp;
+        //str = str.replace('&', '_'); // this should be &amp;
+        str = str.replaceAll("&", "&amp;");
         if (str.indexOf('\"') < 0)
             return sanitizeString(str);
         str = str.replace('\"', '\'');
@@ -889,6 +924,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     public static final String sanitizeXML(String orig) {
         if (orig == null) return "";
         if (orig.indexOf('&') < 0) return orig;
+        if (true) return orig.replaceAll("&", "&amp;");
         StringBuffer rv = new StringBuffer(orig.length()+32);
         for (int i = 0; i < orig.length(); i++) {
             if (orig.charAt(i) == '&')
@@ -900,14 +936,10 @@ public class HTMLRenderer extends EventReceiverImpl {
     }
     public static final String sanitizeXML(StringBuffer orig) {
         if (orig == null) return "";
-        if (orig.indexOf("&") < 0) return orig.toString();
-        for (int i = 0; i < orig.length(); i++) {
-            if (orig.charAt(i) == '&') {
-                orig = orig.replace(i, i+1, "&amp;");
-                i += "&amp;".length();
-            }
-        }
-        return orig.toString();
+        if (orig.indexOf("&") >= 0) 
+            return orig.toString().replaceAll("&", "&amp;");
+        else
+            return orig.toString();
     }
 
     private static final String STYLE_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_";

apps/syndie/java/src/net/i2p/syndie/sml/RSSRenderer.java

@@ -4,6 +4,7 @@ import java.io.*;
 import java.util.*;
 import java.text.SimpleDateFormat;
 import net.i2p.I2PAppContext;
+import net.i2p.client.naming.PetName;
 import net.i2p.data.*;
 import net.i2p.syndie.*;
 import net.i2p.syndie.data.*;
@@ -17,16 +18,21 @@ public class RSSRenderer extends HTMLRenderer {
         super(ctx);
     }
     
+    private static final boolean RSS_EXCERPT_ONLY = false;
+    
     public void render(User user, Archive archive, EntryContainer entry, String urlPrefix, Writer out) throws IOException {
         if (entry == null) return;
-        prepare(user, archive, entry, entry.getEntry().getText(), out, true, false);
+        prepare(user, archive, entry, entry.getEntry().getText(), out, RSS_EXCERPT_ONLY, false);
         
         out.write("   <item>\n");
         out.write("    <title>" + sanitizeXML(sanitizeString((String)_headers.get(HEADER_SUBJECT))) + "</title>\n");
         out.write("    <link>" + urlPrefix + sanitizeXML(getEntryURL()) + "</link>\n");
         out.write("    <guid isPermalink=\"false\">" + urlPrefix + entry.getURI().toString() + "</guid>\n");
         out.write("    <pubDate>" + getRFC822Date(entry.getURI().getEntryId()) + "</pubDate>\n");
-        String author = user.getPetNameDB().getNameByLocation(entry.getURI().getKeyHash().toBase64());
+        PetName pn = user.getPetNameDB().getByLocation(entry.getURI().getKeyHash().toBase64());
+        String author = null;
+        if (pn != null)
+            author = pn.getName();
         if (author == null) {
             BlogInfo info = archive.getBlogInfo(entry.getURI());
             if (info != null)
@@ -192,10 +198,10 @@ public class RSSRenderer extends HTMLRenderer {
             _addresses.add(a);
         if (!continueBody()) { return; }
         if ( (schema == null) || (location == null) ) return;
-        String knownName = null;
+        PetName pn = null;
         if (_user != null)
-            knownName = _user.getPetNameDB().getNameByLocation(location);
-        if (knownName != null) {
+            pn = _user.getPetNameDB().getByLocation(location);
+        if (pn != null) {
             _bodyBuffer.append(sanitizeString(anchorText));
         } else {
             _bodyBuffer.append(sanitizeString(anchorText));
@@ -250,10 +256,10 @@ public class RSSRenderer extends HTMLRenderer {
             for (int i = 0; i < _addresses.size(); i++) {
                 Address a = (Address)_addresses.get(i);
 
-                String knownName = null;
+                PetName pn = null;
                 if (_user != null)
-                    knownName = _user.getPetNameDB().getNameByLocation(a.location);
-                if (knownName == null) {
+                    pn = _user.getPetNameDB().getByLocation(a.location);
+                if (pn == null) {
                     StringBuffer url = new StringBuffer(128);
                     url.append("addresses.jsp?network=");
                     url.append(sanitizeTagParam(a.schema)).append("&location=");

apps/syndie/java/src/net/i2p/syndie/sml/SMLParser.java

@@ -86,6 +86,8 @@ public class SMLParser {
         while (off < len) {
             char c = rawSMLBody.charAt(off);
             if ( (c == NL) || (c == CR) || (c == LF) ) {
+                // we only deal with newlines outside of a tag, since this is a ghetto parser
+                // without a stack, and the tag event is fired only when the tag is completed.
                 if (openTagBegin < 0) {
                     if (begin < off)
                         receiver.receivePlain(rawSMLBody.substring(begin, off));
@@ -93,8 +95,6 @@ public class SMLParser {
                     off++;
                     begin = off;
                     continue;
-                } else {
-                    // ignore NL inside a tag or between tag blocks
                 }
             } else if (c == TAG_BEGIN) {
                 if ( (off + 1 < len) && (TAG_BEGIN == rawSMLBody.charAt(off+1))) {
@@ -142,19 +142,25 @@ public class SMLParser {
                     continue;
                 }
             } else if (c == LT) {
-                if (begin < off)
-                    receiver.receivePlain(rawSMLBody.substring(begin, off));
-                receiver.receiveLT();
-                off++;
-                begin = off;
-                continue;
+                // see above re: newlines inside tags for why we check openTagBegin<0
+                if (openTagBegin < 0) {
+                    if (begin < off)
+                        receiver.receivePlain(rawSMLBody.substring(begin, off));
+                    receiver.receiveLT();
+                    off++;
+                    begin = off;
+                    continue;
+                }
             } else if (c == GT) {
-                if (begin < off)
-                    receiver.receivePlain(rawSMLBody.substring(begin, off));
-                receiver.receiveGT();
-                off++;
-                begin = off;
-                continue;
+                // see above re: newlines inside tags for why we check openTagBegin<0
+                if (openTagBegin < 0) {
+                    if (begin < off)
+                        receiver.receivePlain(rawSMLBody.substring(begin, off));
+                    receiver.receiveGT();
+                    off++;
+                    begin = off;
+                    continue;
+                }
             }
             
             off++;
@@ -443,6 +449,7 @@ public class SMLParser {
         test("A: B\n\n[a   b=\"c\" ]d[/a]");
         
         test("A: B\n\n[b]This[/b] is [i]special[/i][cut]why?[/cut][u]because I say so[/u].\neven if you dont care");
+        test("A: B\n\nHi\n[pre]>foo&bar<>blah!blah\nblah\nblah[/pre]foo![pre]bar[/pre]");
     }
     private static void test(String rawSML) {
         I2PAppContext ctx = I2PAppContext.getGlobalContext();

apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java

@@ -100,10 +100,9 @@ public class ArchiveViewerBean {
         
         for (int i = 0; i < index.getNewestBlogCount(); i++) {
             Hash cur = index.getNewestBlog(i);
-            String knownName = user.getPetNameDB().getNameByLocation(cur.toBase64());
-            PetName pn = null;
-            if (knownName != null) {
-                pn = user.getPetNameDB().get(knownName);
+            PetName pn = user.getPetNameDB().getByLocation(cur.toBase64());
+            String knownName = null;
+            if (pn != null) {
                 knownName = pn.getName();
             }
             if ( (pn != null) && (pn.isMember("Ignore")) )
@@ -128,10 +127,9 @@ public class ArchiveViewerBean {
         Set blogs = index.getUniqueBlogs();
         for (Iterator iter = blogs.iterator(); iter.hasNext(); ) {
             Hash cur = (Hash)iter.next();
-            String knownName = user.getPetNameDB().getNameByLocation(cur.toBase64());
-            PetName pn = null;
-            if (knownName != null) {
-                pn = user.getPetNameDB().get(knownName);
+            PetName pn = user.getPetNameDB().getByLocation(cur.toBase64());
+            String knownName = null;
+            if (pn != null) {
                 knownName = pn.getName();
             }
             if ( (pn != null) && (pn.isMember("Ignore")) )
@@ -494,7 +492,7 @@ public class ArchiveViewerBean {
             PetNameDB db = user.getPetNameDB();
             for (Iterator iter = db.getNames().iterator(); iter.hasNext(); ) {
                 String name = (String)iter.next();
-                PetName pn = db.get(name);
+                PetName pn = db.getByName(name);
                 if ("syndie".equals(pn.getNetwork()) && "syndieblog".equals(pn.getProtocol()) && pn.isMember(group)) {
                     byte pnLoc[] = Base64.decode(pn.getLocation());
                     if (pnLoc != null) {
@@ -517,13 +515,10 @@ public class ArchiveViewerBean {
             BlogURI uri = (BlogURI)uris.get(i);
             Hash k = uri.getKeyHash();
             if (k == null) continue;
-            String pname = user.getPetNameDB().getNameByLocation(k.toBase64());
-            if (pname != null) {
-                PetName pn = user.getPetNameDB().get(pname);
-                if ( (pn != null) && (pn.isMember("Ignore")) ) {
-                    uris.remove(i);
-                    i--;
-                }
+            PetName pn = user.getPetNameDB().getByLocation(k.toBase64());
+            if ( (pn != null) && (pn.isMember("Ignore")) ) {
+                uris.remove(i);
+                i--;
             }
         }
     }

apps/syndie/java/src/net/i2p/syndie/web/PostBean.java

@@ -99,7 +99,7 @@ public class PostBean {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Posted the entry " + uri.toString() + " (archive = " + _archive + ")");
         if ( (uri != null) && BlogManager.instance().authorizeRemote(_user) ) {
-            PetName pn = _user.getPetNameDB().get(_archive);
+            PetName pn = _user.getPetNameDB().getByName(_archive);
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Archive to petname? " + pn + " (protocol: " + (pn != null ? pn.getProtocol() : "") + ")");
             if ( (pn != null) && ("syndiearchive".equals(pn.getProtocol())) ) {

apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java

@@ -6,6 +6,7 @@ import java.util.*;
 import java.util.zip.*;
 import net.i2p.I2PAppContext;
 import net.i2p.client.naming.PetNameDB;
+import net.i2p.client.naming.PetName;
 import net.i2p.data.*;
 import net.i2p.util.EepGet;
 import net.i2p.util.EepGetScheduler;
@@ -61,8 +62,8 @@ public class RemoteArchiveBean {
     
     private boolean ignoreBlog(User user, Hash blog) {
         PetNameDB db = user.getPetNameDB();
-        String name = db.getNameByLocation(blog.toBase64());
-        return ( (name != null) && (db.get(name).isMember("Ignore")) );
+        PetName pn = db.getByLocation(blog.toBase64());
+        return ( (pn!= null) && (pn.isMember("Ignore")) );
     }
     
     public void fetchMetadata(User user, Map parameters) {

apps/syndie/jsp/_bodyindex.jsp

@@ -11,13 +11,14 @@ if (user.getAuthenticated() && (null != request.getParameter("action")) ) {
     group = "Ignore";
   boolean unignore = ("Unignore blog".equals(request.getParameter("action")));
 
-  String name = user.getPetNameDB().getNameByLocation(blog);
+  PetName pn = user.getPetNameDB().getByLocation(blog);
+  String name = null;
+  if (pn != null) name = pn.getName();
   if (name == null)
     name = request.getParameter("name");
   if (name == null)
     name = blog;
   if ( (name != null) && (blog != null) && ( (group != null) || (unignore) ) ) {
-    PetName pn = user.getPetNameDB().get(name);
     if (pn != null) {
       if (unignore)
         pn.removeGroup("Ignore");
@@ -26,7 +27,7 @@ if (user.getAuthenticated() && (null != request.getParameter("action")) ) {
     } else {
       pn = new PetName(name, "syndie", "syndieblog", blog);
       pn.addGroup(group);
-      user.getPetNameDB().set(name, pn);
+      user.getPetNameDB().add(pn);
     }
     BlogManager.instance().saveUser(user);
   }

apps/syndie/jsp/addresses.jsp

@@ -20,7 +20,7 @@ if (!user.getAuthenticated()) {
     String action = request.getParameter("action");
     if ( (action != null) && ("Change".equals(action)) ) {
         String oldPetname = request.getParameter("petname");
-        PetName cur = names.get(oldPetname);
+        PetName cur = names.getByName(oldPetname);
         if (cur != null) {
           cur.setName(request.getParameter("name"));
           cur.setNetwork(request.getParameter("network"));
@@ -28,8 +28,8 @@ if (!user.getAuthenticated()) {
           cur.setIsPublic(null != request.getParameter("isPublic"));
           cur.setLocation(request.getParameter("location"));
           cur.setGroups(request.getParameter("groups"));
-          names.remove(oldPetname);
-          names.set(cur.getName(), cur);
+          names.removeName(oldPetname);
+          names.add(cur);
           names.store(user.getAddressbookLocation());
           if ( ("syndiearchive".equals(cur.getProtocol())) && (BlogManager.instance().authorizeRemote(user)) ) {
             if (null != request.getParameter("scheduleSyndication")) {
@@ -43,7 +43,7 @@ if (!user.getAuthenticated()) {
           %><span class="b_addrMsgOk">Address updated</span><%
         }
     } else if ( (action != null) && ("Add".equals(action)) ) {
-        PetName cur = names.get(request.getParameter("name"));
+        PetName cur = names.getByName(request.getParameter("name"));
         if (cur != null) { %><span class="b_addrMsgErr">Address already exists</span><% } else {
           cur = new PetName();
           cur.setName(request.getParameter("name"));
@@ -52,7 +52,7 @@ if (!user.getAuthenticated()) {
           cur.setIsPublic(null != request.getParameter("isPublic"));
           cur.setLocation(request.getParameter("location"));
           cur.setGroups(request.getParameter("groups"));
-          names.set(cur.getName(), cur);
+          names.add(cur);
           names.store(user.getAddressbookLocation());
           if ( ("syndiearchive".equals(cur.getProtocol())) && (BlogManager.instance().authorizeRemote(user)) ) {
             if (null != request.getParameter("scheduleSyndication")) {
@@ -63,13 +63,13 @@ if (!user.getAuthenticated()) {
           %><span class="b_addrMsgOk">Address added</span><%
         }
     } else if ( (action != null) && ("Delete".equals(action)) ) {
-        PetName cur = names.get(request.getParameter("name"));
+        PetName cur = names.getByName(request.getParameter("name"));
         if (cur != null) { 
           if ( ("syndiearchive".equals(cur.getProtocol())) && (BlogManager.instance().authorizeRemote(user)) ) {
             BlogManager.instance().unscheduleSyndication(cur.getLocation());
             BlogManager.instance().writeConfig();
           }
-          names.remove(cur.getName());
+          names.removeName(cur.getName());
           names.store(user.getAddressbookLocation());
           %><span class="b_addrMsgOk">Address removed</span><%
         }
@@ -90,7 +90,7 @@ if (!user.getAuthenticated()) {
 <%
     StringBuffer buf = new StringBuffer(128);
     for (Iterator iter = sorted.iterator(); iter.hasNext(); ) {
-        PetName name = names.get((String)iter.next());
+        PetName name = names.getByName((String)iter.next());
         buf.append("<tr class=\"b_addrDetail\"><form action=\"addresses.jsp\" method=\"POST\">");
         buf.append("<input type=\"hidden\" name=\"petname\" value=\"").append(name.getName()).append("\" />");
         buf.append("<td class=\"b_addrName\"><input class=\"b_addrName\" type=\"text\" size=\"20\" name=\"name\" value=\"").append(name.getName()).append("\" /></td>");

apps/syndie/jsp/post.jsp

@@ -66,7 +66,7 @@ if (!user.getAuthenticated()) {
           PetNameDB db = user.getPetNameDB();
           if (entryHeaders == null) entryHeaders = "";
           for (Iterator iter = db.getNames().iterator(); iter.hasNext(); ) {
-            PetName pn = db.get((String)iter.next());
+            PetName pn = db.getByName((String)iter.next());
             if ( (pn != null) && (pn.getIsPublic()) ) {
               entryHeaders = entryHeaders + '\n' + HTMLRenderer.HEADER_PETNAME + ": " + 
                              pn.getName() + "\t" + pn.getNetwork() + "\t" + pn.getProtocol() + "\t" + pn.getLocation();
@@ -107,7 +107,7 @@ PetNameDB db = user.getPetNameDB();
 TreeSet names = new TreeSet();
 for (Iterator iter = db.getNames().iterator(); iter.hasNext(); ) {
   String name = (String)iter.next();
-  PetName pn = db.get(name);
+  PetName pn = db.getByName(name);
   if ("syndiearchive".equals(pn.getProtocol()))
     names.add(pn.getName());
 }

apps/syndie/jsp/remote.jsp

@@ -29,8 +29,8 @@ if (!BlogManager.instance().authorizeRemote(user)) {
   <input class="b_remoteChooserHost" type="text" size="10" name="proxyhost" value="<%=BlogManager.instance().getDefaultProxyHost()%>" />
   <input class="b_remoteChooserPort" type="text" size="4" name="proxyport" value="<%=BlogManager.instance().getDefaultProxyPort()%>" /><br />
 <span class="b_remoteChooserField">Bookmarked archives:</span> <select class="b_remoteChooserPN" name="archivepetname"><option value="">Custom location</option><%
-for (Iterator iter = user.getPetNameDB().getNames().iterator(); iter.hasNext(); ) {
-  PetName pn = user.getPetNameDB().get((String)iter.next());
+for (Iterator iter = user.getPetNameDB().iterator(); iter.hasNext(); ) {
+  PetName pn = (PetName)iter.next();
   if ("syndiearchive".equals(pn.getProtocol())) {
     %><option value="<%=HTMLRenderer.sanitizeTagParam(pn.getName())%>"><%=HTMLRenderer.sanitizeString(pn.getName())%></option><%
   }
@@ -45,7 +45,7 @@ for (Iterator iter = user.getPetNameDB().getNames().iterator(); iter.hasNext();
     String location = request.getParameter("location");
     String pn = request.getParameter("archivepetname");
     if ( (pn != null) && (pn.trim().length() > 0) ) {
-      PetName pnval = user.getPetNameDB().get(pn);
+      PetName pnval = user.getPetNameDB().getByName(pn);
       if (pnval != null) location = pnval.getLocation();
     }
     remote.fetchIndex(user, request.getParameter("schema"), location, request.getParameter("proxyhost"), request.getParameter("proxyport"));

build.xml

@@ -29,7 +29,7 @@
         <ant dir="apps/addressbook/" target="war" />
         <ant dir="apps/susimail/" target="war" />
         <ant dir="apps/susidns/src" target="all" />
-        <ant dir="apps/syndie/java/" target="jar" /> <!-- not pushed in the update... yet -->
+        <ant dir="apps/syndie/java/" target="jar" />
     </target>
     <target name="buildWEB">
         <ant dir="apps/jetty" target="fetchJettylib" />
@@ -89,7 +89,10 @@
         <copy file="apps/susidns/src/susidns.war" todir="build/" />
         <copy file="apps/syndie/syndie.war" todir="build/" />
         <copy file="apps/syndie/java/build/syndie.jar" todir="build/" />
+        <copy file="apps/syndie/java/build/sucker.jar" todir="build/" />
         <copy file="apps/syndie/syndie.war" todir="build/" />
+        <copy file="apps/jdom/jdom.jar" todir="build/" />
+        <copy file="apps/rome/rome-0.7.jar" todir="build/" />
     </target>
     <target name="javadoc">
         <mkdir dir="./build" />
@@ -211,6 +214,9 @@
         <copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
         <copy file="build/sam.jar" todir="pkg-temp/lib/" />
         <copy file="build/systray.jar" todir="pkg-temp/lib" />
+        <copy file="build/jdom.jar" todir="pkg-temp/lib" />
+        <copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
+        <copy file="build/sucker.jar" todir="pkg-temp/lib" />
         <copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
         <copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
         <copy file="apps/systray/java/lib/systray4j.jar" todir="pkg-temp/lib" />
@@ -310,6 +316,12 @@
         <copy file="build/sam.jar" todir="pkg-temp/lib/" />
         <copy file="build/router.jar" todir="pkg-temp/lib/" />
         <copy file="build/routerconsole.jar" todir="pkg-temp/lib/" />
+        <!-- for the 0.6.1.3 release, push JDOM and ROME.  it can be
+             removed later updates (once people on earlier builds upgrade -->
+        <copy file="build/jdom.jar" todir="pkg-temp/lib" />
+        <copy file="build/rome-0.7.jar" todir="pkg-temp/lib" />
+        <copy file="build/sucker.jar" todir="pkg-temp/lib" />
+        
         <copy file="i2p.exe" todir="pkg-temp/" failonerror="false" />
         <copy file="installer/resources/runplain.sh" todir="pkg-temp/" />
         

core/java/src/net/i2p/CoreVersion.java

@@ -14,8 +14,8 @@ package net.i2p;
  *
  */
 public class CoreVersion {
-    public final static String ID = "$Revision: 1.44 $ $Date: 2005/10/01 14:20:09 $";
-    public final static String VERSION = "0.6.1.2";
+    public final static String ID = "$Revision: 1.45 $ $Date: 2005/10/07 15:19:09 $";
+    public final static String VERSION = "0.6.1.3";
 
     public static void main(String args[]) {
         System.out.println("I2P Core version: " + VERSION);

core/java/src/net/i2p/client/naming/PetNameDB.java

@@ -16,14 +16,20 @@ public class PetNameDB {
         _names = Collections.synchronizedMap(new HashMap());
     }
 
-    public PetName get(String name) { return (PetName)_names.get(name); } 
-    public boolean exists(String name) { return _names.containsKey(name); }
-    public void set(String name, PetName pn) { _names.put(name, pn); }
-    public void remove(String name) { _names.remove(name); }
+    public PetName getByName(String name) { return (PetName)_names.get(name.toLowerCase()); }
+    public void add(PetName pn) { _names.put(pn.getName().toLowerCase(), pn); }
+    public void clear() { _names.clear(); }
+    public boolean contains(PetName pn) { return _names.containsValue(pn); }
+    public boolean containsName(String name) { return _names.containsKey(name.toLowerCase()); }
+    public boolean isEmpty() { return _names.isEmpty(); }
+    public Iterator iterator() { return new LinkedList(_names.values()).iterator(); }
+    public void remove(PetName pn) { _names.values().remove(pn); }
+    public void removeName(String name) { _names.remove(name.toLowerCase()); }
+    public int size() { return _names.size(); }
     public Set getNames() { return new HashSet(_names.keySet()); }
     public List getGroups() {
         List rv = new ArrayList();
-        for (Iterator iter = new HashSet(_names.values()).iterator(); iter.hasNext(); ) {
+        for (Iterator iter = iterator(); iter.hasNext(); ) {
             PetName name = (PetName)iter.next();
             for (int i = 0; i < name.getGroupCount(); i++)
                 if (!rv.contains(name.getGroup(i)))
@@ -32,13 +38,13 @@ public class PetNameDB {
         return rv;
     }
     
-    public String getNameByLocation(String location) { 
+    public PetName getByLocation(String location) { 
         if (location == null) return null;
         synchronized (_names) {
-            for (Iterator iter = _names.values().iterator(); iter.hasNext(); ) {
+            for (Iterator iter = iterator(); iter.hasNext(); ) {
                 PetName name = (PetName)iter.next();
                 if ( (name.getLocation() != null) && (name.getLocation().trim().equals(location.trim())) )
-                    return name.getName();
+                    return name;
             }
         }
         return null;
@@ -55,7 +61,7 @@ public class PetNameDB {
             while ( (line = in.readLine()) != null) {
                 PetName name = new PetName(line);
                 if (name.getName() != null)
-                    _names.put(name.getName(), name);
+                    add(name);
             }
         } finally {
             in.close();
@@ -66,8 +72,8 @@ public class PetNameDB {
         Writer out = null;
         try {
             out = new OutputStreamWriter(new FileOutputStream(location), "UTF-8");
-            for (Iterator names = getNames().iterator(); names.hasNext(); ) {
-                PetName name = get((String)names.next());
+            for (Iterator iter = iterator(); iter.hasNext(); ) {
+                PetName name = (PetName)iter.next();
                 if (name != null)
                     out.write(name.toString() + "\n");
             }

core/java/src/net/i2p/client/naming/PetNameNamingService.java

@@ -51,7 +51,7 @@ public class PetNameNamingService extends NamingService {
     }
 
     public Destination lookup(String hostname) {
-        PetName name = _petnameDb.get(hostname);
+        PetName name = _petnameDb.getByName(hostname);
         if (name != null && name.getNetwork().equalsIgnoreCase("i2p")) {
             return lookupBase64(name.getLocation());
         } else {
@@ -60,6 +60,6 @@ public class PetNameNamingService extends NamingService {
     }
 
     public String reverseLookup(Destination dest) {
-        return _petnameDb.getNameByLocation(dest.toBase64());
+        return _petnameDb.getByLocation(dest.toBase64()).getName();
     }
 }

core/java/src/net/i2p/crypto/DHSessionKeyBuilder.java

@@ -66,11 +66,13 @@ public class DHSessionKeyBuilder {
     public final static String PROP_DH_PRECALC_MAX = "crypto.dh.precalc.max";
     public final static String PROP_DH_PRECALC_DELAY = "crypto.dh.precalc.delay";
     public final static String DEFAULT_DH_PRECALC_MIN = "5";
-    public final static String DEFAULT_DH_PRECALC_MAX = "10";
-    public final static String DEFAULT_DH_PRECALC_DELAY = "1000";
+    public final static String DEFAULT_DH_PRECALC_MAX = "50";
+    public final static String DEFAULT_DH_PRECALC_DELAY = "10000";
 
     static {
         I2PAppContext ctx = _context;
+        ctx.statManager().createRateStat("crypto.dhGeneratePublicTime", "How long it takes to create x and X", "Encryption", new long[] { 60*1000, 5*60*1000, 60*60*1000 });
+        ctx.statManager().createRateStat("crypto.dhCalculateSessionTime", "How long it takes to create the session key", "Encryption", new long[] { 60*1000, 5*60*1000, 60*60*1000 });        
         try {
             int val = Integer.parseInt(ctx.getProperty(PROP_DH_PRECALC_MIN, DEFAULT_DH_PRECALC_MIN));
             MIN_NUM_BUILDERS = val;
@@ -225,11 +227,12 @@ public class DHSessionKeyBuilder {
      *
      */
     public BigInteger generateMyValue() {
-        long start = Clock.getInstance().now();
+        long start = System.currentTimeMillis();
         _myPrivateValue = new NativeBigInteger(2048, RandomSource.getInstance());
         BigInteger myValue = CryptoConstants.elgg.modPow(_myPrivateValue, CryptoConstants.elgp);
-        long end = Clock.getInstance().now();
+        long end = System.currentTimeMillis();
         long diff = end - start;
+        _context.statManager().addRateData("crypto.dhGeneratePublicTime", diff, diff);
         if (diff > 1000) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Took more than a second (" + diff + "ms) to generate local DH value");
@@ -339,7 +342,7 @@ public class DHSessionKeyBuilder {
      *
      */
     private final SessionKey calculateSessionKey(BigInteger myPrivateValue, BigInteger publicPeerValue) {
-        long start = Clock.getInstance().now();
+        long start = System.currentTimeMillis();
         SessionKey key = new SessionKey();
         BigInteger exchangedKey = publicPeerValue.modPow(myPrivateValue, CryptoConstants.elgp);
         byte buf[] = exchangedKey.toByteArray();
@@ -361,8 +364,10 @@ public class DHSessionKeyBuilder {
                 _log.debug("Storing " + remaining.length + " bytes from the end of the DH exchange");
         }
         key.setData(val);
-        long end = Clock.getInstance().now();
+        long end = System.currentTimeMillis();
         long diff = end - start;
+        
+        _context.statManager().addRateData("crypto.dhCalculateSessionTime", diff, diff);
         if (diff > 1000) {
             if (_log.shouldLog(Log.WARN)) _log.warn("Generating session key took too long (" + diff + " ms");
         } else {
@@ -490,10 +495,12 @@ public class DHSessionKeyBuilder {
                 curSize = startSize;
                 while (curSize < _minSize) {
                     while (curSize < _maxSize) {
+                        long curStart = System.currentTimeMillis();
                         curSize = addBuilder(precalc(curSize));
+                        long curCalc = System.currentTimeMillis() - curStart;
                         // for some relief...
                         try {
-                            Thread.sleep(CALC_DELAY);
+                            Thread.sleep(CALC_DELAY + curCalc * 10);
                         } catch (InterruptedException ie) { // nop
                         }
                     }

core/java/src/net/i2p/stat/Rate.java

@@ -454,7 +454,10 @@ public class Rate {
     }
 
     private final static long now() {
-        return Clock.getInstance().now();
+        // "event time" is in the stat log (and uses Clock).
+        // we just want sequential and stable time here, so use the OS time, since it doesn't
+        // skew periodically
+        return System.currentTimeMillis(); //Clock.getInstance().now();
     }
 
     public static void main(String args[]) {

core/java/src/net/i2p/util/EepGet.java

@@ -487,12 +487,14 @@ public class EepGet {
             _log.debug("Status line: [" + line + "]");
         StringTokenizer tok = new StringTokenizer(line, " ");
         if (!tok.hasMoreTokens()) {
-            System.err.println("ERR: status "+  line);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("ERR: status "+  line);
             return -1;
         }
         String protocol = tok.nextToken(); // ignored
         if (!tok.hasMoreTokens()) {
-            System.err.println("ERR: status "+  line);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("ERR: status "+  line);
             return -1;
         }
         String rc = tok.nextToken();

history.txt

@@ -1,4 +1,73 @@
-$Id: history.txt,v 1.284 2005/10/07 05:23:01 jrandom Exp $
+$Id: history.txt,v 1.294 2005/10/13 21:15:40 jrandom Exp $
+
+* 2005-10-14  0.6.1.3 released
+
+2005-10-14  jrandom
+    * Added a key explaining peers.jsp a bit (thanks tethra!)
+
+2005-10-13  dust
+    * Bundled dust's Sucker for pulling RSS/Atom content into SML, which can
+      then be injected into Syndie with the Syndie CLI.
+    * Bundled ROME and JDOM (BSD and Apache licensed, respectively) for
+      RSS/Atom parsing.
+
+2005-10-13  jrandom
+    * SSU retransmission choke bugfix (== != !=)
+    * Include initial transmissions in the retransmission choke, so that
+      if we are already retransmitting a message, we won't send anything
+      to that peer other than that message (or ACKs, if necessary)
+
+2005-10-12  jrandom
+    * Choke SSU retransmissions to a peer while there is already a
+      retransmission in flight to them.  This currently lets other initial
+      transmissions through, since packet loss is often sporadic, but maybe
+      this should block initial transmissions as well?
+    * Display the retransmission bytes stat on peers.jsp (thanks bar!)
+    * Filter QUIT messages in the I2PTunnelIRCClient proxy
+
+2005-10-11  jrandom
+    * Piggyback the SSU partial ACKs with data packets.  This is backwards
+      compatible.
+    * Syndie RSS renderer bugfix, plus now include the full entry instead of
+      just the blurb before the cut.
+
+2005-10-11  jrandom
+    * Piggyback the SSU explicit ACKs with data packets (partial ACKs aren't
+      yet piggybacked).  This is backwards compatible.
+    * SML parser cleanup in Syndie
+
+2005-10-10  dust
+    * Implemented a new I2PTunnelIRCClient which locally filters inbound and
+      outbound IRC commands for anonymity and security purposes, removing all
+      CTCP messages except ACTION, as well as stripping the hostname from the
+      USER message (while leaving the nick and 'full name').  The IRC proxy
+      doesn't use this by default, but you can enable it by creating a new 
+      "IRC proxy" tunnel on the web interface, or by changing the tunnel type
+      to "ircclient" in i2ptunnel.config.
+
+2005-10-10  jrandom
+    * I2PTunnel http client config cleanup and stats
+    * Minor SSU congestion tweaks and stats
+    * Reduced netDb exploration period
+
+2005-10-09  jrandom
+    * Syndie CLI cleanup for simpler CLI posting.  Usage shown with
+      java -jar lib/syndie.jar
+    * Beginnings of the Syndie logging cleanup
+    * Delete corrupt Syndie posts
+
+2005-10-09  jrandom
+    * Now that the streaming lib works reasonably, set the default inactivity 
+      event to send a 0 byte keepalive payload, rather than disconnecting the
+      stream.  This should cut the irc netsplits and help out with other long
+      lived streams.  The default timeout is now less than the old timeout as
+      well, so the keepalive will be sent before earlier builds fire their
+      fatal timeouts.
+
+2005-10-08  jrandom
+    * Use the OS clock for stat timing, since it doesn't jump around (though
+      still use the NTP'ed clock for display)
+    * Added new DH stats
 
 * 2005-10-07  0.6.1.2 released
 

initialNews.xml

@@ -1,5 +1,5 @@
-<i2p.news date="$Date: 2005/10/01 14:20:10 $">
- <i2p.release version="0.6.1.2" date="2005/10/07" minVersion="0.6"
+<i2p.news date="$Date: 2005/10/07 15:19:04 $">
+ <i2p.release version="0.6.1.3" date="2005/10/14" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"
               anonannouncement="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/pipermail/i2p/2005-September/000878.html" 

installer/install.xml

@@ -4,7 +4,7 @@
 
     <info>
         <appname>i2p</appname>
-        <appversion>0.6.1.2</appversion>
+        <appversion>0.6.1.3</appversion>
         <authors>
             <author name="I2P" email="support@i2p.net"/>
         </authors>

installer/resources/i2ptunnel.config

@@ -16,7 +16,7 @@ tunnel.0.startOnLoad=true
 # irc
 tunnel.1.name=ircProxy
 tunnel.1.description=IRC proxy to access the anonymous irc net
-tunnel.1.type=client
+tunnel.1.type=ircclient
 tunnel.1.sharedClient=true
 tunnel.1.interface=127.0.0.1
 tunnel.1.listenPort=6668

news.xml

@@ -1,5 +1,5 @@
-<i2p.news date="$Date: 2005/10/01 14:20:10 $">
- <i2p.release version="0.6.1.2" date="2005/10/07" minVersion="0.6"
+<i2p.news date="$Date: 2005/10/07 18:45:48 $">
+ <i2p.release version="0.6.1.3" date="2005/10/14" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"
               anonannouncement="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/pipermail/i2p/2005-September/000878.html" 
@@ -11,5 +11,5 @@
             publiclogs="http://www.i2p.net/meeting141" />
 &#149; Tip: if an eepsite isn't reachable, it may not be online - <a href="http://orion.i2p/">orion</a> or 
        <a href="http://polecat.i2p/i2psurvey/">polecat</a> tells whether they can reach the site as well<br />
-&#149; I2Phex users should keep track of the progress on the <a href="http://forum.i2p/viewforum.i2p?f=25">forum</a><br />
+&#149; I2Phex users should keep track of the progress on the <a href="http://forum.i2p/viewforum.php?f=25">forum</a><br />
 </i2p.news>

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,8 +15,8 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.259 $ $Date: 2005/10/07 05:23:01 $";
-    public final static String VERSION = "0.6.1.2";
+    public final static String ID = "$Revision: 1.269 $ $Date: 2005/10/13 21:15:40 $";
+    public final static String VERSION = "0.6.1.3";
     public final static long BUILD = 0;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);

router/java/src/net/i2p/router/networkdb/kademlia/ExploreJob.java

@@ -27,8 +27,8 @@ class ExploreJob extends SearchJob {
     private Log _log;
     private PeerSelector _peerSelector;
     
-    /** how long each exploration should run for (currently a trivial 20 seconds) */
-    private static final long MAX_EXPLORE_TIME = 20*1000;
+    /** how long each exploration should run for (currently a trivial 10 seconds) */
+    private static final long MAX_EXPLORE_TIME = 10*1000;
     
     /** how many of the peers closest to the key being explored do we want to explicitly say "dont send me this"? */
     private static final int NUM_CLOSEST_TO_IGNORE = 3;

router/java/src/net/i2p/router/transport/tcp/TCPListener.java

@@ -60,6 +60,9 @@ class TCPListener {
         _transport = transport;
         _pendingSockets = new ArrayList(10);
         _handlers = new ArrayList(CONCURRENT_HANDLERS);
+        _context.statManager().createRateStat("tcp.conReceiveOK", "How long does it take to receive a valid connection", "TCP", new long[] { 60*1000, 5*60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tcp.conReceiveFail", "How long does it take to receive a failed connection", "TCP", new long[] { 60*1000, 5*60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tcp.conUnhandled", "How often do we receive a connection but take too long on other ones to handle it", "TCP", new long[] { 60*1000, 5*60*1000, 10*60*1000 });
     }
         
     /** Make sure we are listening per the transport's config */
@@ -230,6 +233,7 @@ class TCPListener {
                 removed = _pendingSockets.remove(_cur);
             }
             if (removed) {
+                _context.statManager().addRateData("tcp.conUnhandled", 1, 0);
                 // handlers hadn't taken it yet, so close it
                 if (_log.shouldLog(Log.WARN))
                     _log.warn("Closing unhandled socket " + _cur);
@@ -294,7 +298,13 @@ class TCPListener {
             ConnectionHandler ch = new ConnectionHandler(_context, _transport, _socket);
             TCPConnection con = null;
             try {
+                long before = System.currentTimeMillis();
                 con = ch.receiveConnection();
+                long duration = System.currentTimeMillis() - before;
+                if (con != null)
+                    _context.statManager().addRateData("tcp.conReceiveOK", duration, duration);
+                else
+                    _context.statManager().addRateData("tcp.conReceiveFail", duration, duration);
             } catch (Exception e) {
                 _log.log(Log.CRIT, "Unhandled exception receiving a connection on " + _socket, e);
             }

router/java/src/net/i2p/router/transport/udp/ACKSender.java

@@ -34,6 +34,7 @@ public class ACKSender implements Runnable {
         _context.statManager().createRateStat("udp.sendACKCount", "how many ack messages were sent to a peer", "udp", new long[] { 60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.ackFrequency", "how long ago did we send an ACK to this peer?", "udp", new long[] { 60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.sendACKRemaining", "when we ack a peer, how many peers are left waiting to ack?", "udp", new long[] { 60*1000, 60*60*1000 });
+        _context.statManager().createRateStat("udp.abortACK", "How often do we schedule up an ACK send only to find it had already been sent (through piggyback)?", "udp", new long[] { 60*1000, 60*60*1000 });
     }
     
     public void ackPeer(PeerState peer) {
@@ -121,6 +122,8 @@ public class ACKSender implements Runnable {
                             _log.warn("Rerequesting ACK for peer " + peer);
                         ackPeer(peer);
                     }
+                } else {
+                    _context.statManager().addRateData("udp.abortACK", 1, 0);
                 }
             }
         }

router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java

@@ -49,6 +49,7 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
         _context.statManager().createRateStat("udp.ignoreRecentDuplicate", "Take note that we received a packet for a recently completed message", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.receiveMessagePeriod", "How long it takes to pull the message fragments out of a packet", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.receiveACKPeriod", "How long it takes to pull the ACKs out of a packet", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.receivePiggyback", "How many acks were included in a packet with data fragments (time == # data fragments)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
     }
     
     public void startup() { 
@@ -75,14 +76,16 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
      */
     public void receiveData(PeerState from, UDPPacketReader.DataReader data) {
         long beforeMsgs = _context.clock().now();
-        receiveMessages(from, data);
+        int fragmentsIncluded = receiveMessages(from, data);
         long afterMsgs = _context.clock().now();
-        receiveACKs(from, data);
+        int acksIncluded = receiveACKs(from, data);
         long afterACKs = _context.clock().now();
         
         from.packetReceived();
         _context.statManager().addRateData("udp.receiveMessagePeriod", afterMsgs-beforeMsgs, afterACKs-beforeMsgs);
         _context.statManager().addRateData("udp.receiveACKPeriod", afterACKs-afterMsgs, afterACKs-beforeMsgs);
+        if ( (fragmentsIncluded > 0) && (acksIncluded > 0) )
+            _context.statManager().addRateData("udp.receivePiggyback", acksIncluded, fragmentsIncluded);
     }
     
     /**
@@ -90,10 +93,11 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
      * Along the way, if any state expires, or a full message arrives, move it
      * appropriately.
      *
+     * @return number of data fragments included
      */
-    private void receiveMessages(PeerState from, UDPPacketReader.DataReader data) {
+    private int receiveMessages(PeerState from, UDPPacketReader.DataReader data) {
         int fragments = data.readFragmentCount();
-        if (fragments <= 0) return;
+        if (fragments <= 0) return fragments;
         Hash fromPeer = from.getRemotePeer();
             
         Map messages = from.getInboundMessages();
@@ -170,13 +174,16 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
                     break;
             }
         }
+        return fragments;
     }
     
-    private void receiveACKs(PeerState from, UDPPacketReader.DataReader data) {
+    private int receiveACKs(PeerState from, UDPPacketReader.DataReader data) {
+        int rv = 0;
         if (data.readACKsIncluded()) {
             int fragments = 0;
             long acks[] = data.readACKs();
             if (acks != null) {
+                rv += acks.length;
                 _context.statManager().addRateData("udp.receivedACKs", acks.length, 0);
                 //_context.statManager().getStatLog().addData(from.getRemoteHostId().toString(), "udp.peer.receiveACKCount", acks.length, 0);
 
@@ -192,6 +199,7 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
         if (data.readACKBitfieldsIncluded()) {
             ACKBitfield bitfields[] = data.readACKBitfields();
             if (bitfields != null) {
+                rv += bitfields.length;
                 //_context.statManager().getStatLog().addData(from.getRemoteHostId().toString(), "udp.peer.receivePartialACKCount", bitfields.length, 0);
 
                 for (int i = 0; i < bitfields.length; i++) {
@@ -205,5 +213,6 @@ public class InboundMessageFragments /*implements UDPTransport.PartialACKSource
             from.ECNReceived();
         else
             from.dataReceived();
+        return rv;
     }
 }

router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java

@@ -1,7 +1,10 @@
 package net.i2p.router.transport.udp;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import net.i2p.data.Hash;
 import net.i2p.data.RouterInfo;
@@ -36,6 +39,13 @@ public class OutboundMessageFragments {
     private PacketBuilder _builder;
     /** if we can handle more messages explicitly, set this to true */
     private boolean _allowExcess;
+    private volatile long _packetsRetransmitted;
+    /** 
+     * Map of peer to OutboundMessageState for messages being retransmitted, to
+     * keep bad peers from bursting too much due to congestion/outage.  This
+     * should only be accessed when holding the lock on _activeMessages.
+     */
+    private Map _retransmitters;
     
     private static final int MAX_ACTIVE = 64;
     // don't send a packet more than 10 times
@@ -47,6 +57,7 @@ public class OutboundMessageFragments {
         _transport = transport;
         _throttle = throttle;
         _activeMessages = new ArrayList(MAX_ACTIVE);
+        _retransmitters = new HashMap(MAX_ACTIVE);
         _nextPacketMessage = 0;
         _builder = new PacketBuilder(ctx, transport);
         _alive = true;
@@ -61,7 +72,12 @@ public class OutboundMessageFragments {
         _context.statManager().createRateStat("udp.sendRejected", "What volley are we on when the peer was throttled (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.partialACKReceived", "How many fragments were partially ACKed (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendSparse", "How many fragments were partially ACKed and hence not resent (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.sendPiggyback", "How many acks were piggybacked on a data packet (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.sendPiggybackPartial", "How many partial acks were piggybacked on a data packet (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.activeDelay", "How often we wait blocking on the active queue", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.packetsRetransmitted", "How many packets have been retransmitted (lifetime) when a burst of packets are retransmitted (period == packets transmitted, lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.peerPacketsRetransmitted", "How many packets have been retransmitted to the peer (lifetime) when a burst of packets are retransmitted (period == packets transmitted, lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.blockedRetransmissions", "How packets have been transmitted to the peer when we blocked a retransmission to them?", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
     }
     
     public void startup() { _alive = true; }
@@ -155,6 +171,7 @@ public class OutboundMessageFragments {
                 PeerState peer = state.getPeer();
                 if (state.isComplete()) {
                     _activeMessages.remove(i);
+                    locked_removeRetransmitter(state);
                     _transport.succeeded(state.getMessage());
                     if ( (peer != null) && (peer.getSendWindowBytesRemaining() > 0) )
                         _throttle.unchoke(peer.getRemotePeer());
@@ -167,6 +184,7 @@ public class OutboundMessageFragments {
                     i--;
                 } else if (state.isExpired()) {
                     _activeMessages.remove(i);
+                    locked_removeRetransmitter(state);
                     _context.statManager().addRateData("udp.sendFailed", state.getPushCount(), state.getLifetime());
 
                     if (state.getMessage() != null) {
@@ -188,6 +206,7 @@ public class OutboundMessageFragments {
                     i--;
                 } else if (state.getPushCount() > MAX_VOLLEYS) {
                     _activeMessages.remove(i);
+                    locked_removeRetransmitter(state);
                     _context.statManager().addRateData("udp.sendAggressiveFailed", state.getPushCount(), state.getLifetime());
                     //if (state.getPeer() != null)
                     //    state.getPeer().congestionOccurred();
@@ -215,6 +234,29 @@ public class OutboundMessageFragments {
         } // end synchronized
     }
     
+    /**
+     * Remove the block on retransmissions to the peer if and only if the given
+     * message is the current "retransmitter" for it.
+     *
+     */
+    private void locked_removeRetransmitter(OutboundMessageState state) {
+        PeerState curPeer = state.getPeer();
+        if (curPeer == null) {
+            for (Iterator iter = _retransmitters.keySet().iterator(); iter.hasNext(); ) {
+                PeerState cpeer = (PeerState)iter.next();
+                OutboundMessageState cstate = (OutboundMessageState)_retransmitters.get(cpeer);
+                if (cstate == state) {
+                    iter.remove();
+                    break;
+                }
+            }
+        } else {
+            OutboundMessageState remState = (OutboundMessageState)_retransmitters.get(curPeer);
+            if (remState == state)
+                _retransmitters.remove(curPeer);
+        }
+    }
+    
     private static final long SECOND_MASK = 1023l;
 
     /**
@@ -248,6 +290,7 @@ public class OutboundMessageFragments {
                             if (peer == null) {
                                 // peer disconnected
                                 _activeMessages.remove(cur);
+                                locked_removeRetransmitter(state);
                                 _transport.failed(state);
                                 if (_log.shouldLog(Log.WARN))
                                     _log.warn("Peer disconnected for " + state);
@@ -301,6 +344,14 @@ public class OutboundMessageFragments {
                     _log.info("Fragmenting " + state);
             }
 
+            OutboundMessageState curRetransMsg = (OutboundMessageState)_retransmitters.get(peer);
+            if ( (curRetransMsg != null) && (curRetransMsg != state) ) {
+                // choke it, since there's already another message retransmitting to this
+                // peer.
+                _context.statManager().addRateData("udp.blockedRetransmissions", peer.getPacketsRetransmitted(), peer.getPacketsTransmitted());
+                return false;
+            }
+
             int size = state.getUnackedSize();
             if (peer.allocateSendingBytes(size)) {
                 if (_log.shouldLog(Log.INFO))
@@ -311,6 +362,8 @@ public class OutboundMessageFragments {
                               + " for message " + state.getMessageId() + ": " + state);
 
                 if (state.getPushCount() > 0) {
+                    _retransmitters.put(peer, state);
+
                     int fragments = state.getFragmentCount();
                     int toSend = 0;
                     for (int i = 0; i < fragments; i++) {
@@ -319,6 +372,9 @@ public class OutboundMessageFragments {
                     }
 
                     peer.messageRetransmitted(toSend);
+                    _packetsRetransmitted += toSend; // lifetime for the transport
+                    _context.statManager().addRateData("udp.peerPacketsRetransmitted", peer.getPacketsRetransmitted(), peer.getPacketsTransmitted());
+                    _context.statManager().addRateData("udp.packetsRetransmitted", _packetsRetransmitted, peer.getPacketsTransmitted());
                     if (_log.shouldLog(Log.WARN))
                         _log.warn("Retransmitting " + state + " to " + peer);
                     _context.statManager().addRateData("udp.sendVolleyTime", state.getLifetime(), toSend);
@@ -354,16 +410,38 @@ public class OutboundMessageFragments {
             if (fragments < 0)
                 return null;
             
+            // ok, simplest possible thing is to always tack on the bitfields if
+            List msgIds = peer.getCurrentFullACKs();
+            List partialACKBitfields = new ArrayList();
+            peer.fetchPartialACKs(partialACKBitfields);
+            int piggybackedPartialACK = partialACKBitfields.size();
+            List remaining = new ArrayList(msgIds);
             int sparseCount = 0;
             UDPPacket rv[] = new UDPPacket[fragments]; //sparse
             for (int i = 0; i < fragments; i++) {
                 if (state.needsSending(i))
-                    rv[i] = _builder.buildPacket(state, i, peer);
+                    rv[i] = _builder.buildPacket(state, i, peer, remaining, partialACKBitfields);
                 else
                     sparseCount++;
             }
+            
+            int piggybackedAck = 0;
+            if (msgIds.size() != remaining.size()) {
+                for (int i = 0; i < msgIds.size(); i++) {
+                    Long id = (Long)msgIds.get(i);
+                    if (!remaining.contains(id)) {
+                        peer.removeACKMessage(id);
+                        piggybackedAck++;
+                    }
+                }
+            }
+            
             if (sparseCount > 0)
                 _context.statManager().addRateData("udp.sendSparse", sparseCount, state.getLifetime());
+            if (piggybackedAck > 0)
+                _context.statManager().addRateData("udp.sendPiggyback", piggybackedAck, state.getLifetime());
+            if (piggybackedPartialACK - partialACKBitfields.size() > 0)
+                _context.statManager().addRateData("udp.sendPiggybackPartial", piggybackedPartialACK - partialACKBitfields.size(), state.getLifetime());
             if (_log.shouldLog(Log.INFO))
                 _log.info("Building packet for " + state + " to " + peer + " with sparse count: " + sparseCount);
             peer.packetsTransmitted(fragments - sparseCount);
@@ -412,6 +490,7 @@ public class OutboundMessageFragments {
                         if (_nextPacketMessage < 0)
                             _nextPacketMessage = 0;
                     }
+                    locked_removeRetransmitter(state);
                     break;
                 } else {
                     state = null;
@@ -484,6 +563,7 @@ public class OutboundMessageFragments {
                                 _nextPacketMessage = 0;
                         }
                     }
+                    locked_removeRetransmitter(state);
                     break;
                 } else {
                     state = null;

router/java/src/net/i2p/router/transport/udp/PacketBuilder.java

@@ -38,11 +38,23 @@ public class PacketBuilder {
     }
     
     public UDPPacket buildPacket(OutboundMessageState state, int fragment, PeerState peer) {
+        return buildPacket(state, fragment, peer, null, null);
+    }
+    /**
+     * @param ackIdsRemaining list of messageIds (Long) that should be acked by this packet.  
+     *                        The list itself is passed by reference, and if a messageId is
+     *                        included, it should be removed from the list.
+     * @param partialACKsRemaining list of messageIds (ACKBitfield) that should be acked by this packet.  
+     *                        The list itself is passed by reference, and if a messageId is
+     *                        included, it should be removed from the list.
+     */
+    public UDPPacket buildPacket(OutboundMessageState state, int fragment, PeerState peer, List ackIdsRemaining, List partialACKsRemaining) {
         UDPPacket packet = UDPPacket.acquire(_context);
         
         byte data[] = packet.getPacket().getData();
         Arrays.fill(data, 0, data.length, (byte)0x0);
-        int off = UDPPacket.MAC_SIZE + UDPPacket.IV_SIZE;
+        int start = UDPPacket.MAC_SIZE + UDPPacket.IV_SIZE;
+        int off = start;
         
         // header
         data[off] |= (UDPPacket.PAYLOAD_TYPE_DATA << 4);
@@ -56,7 +68,54 @@ public class PacketBuilder {
         
         // just always ask for an ACK for now...
         data[off] |= UDPPacket.DATA_FLAG_WANT_REPLY;
+        // we should in theory only include explicit ACKs if the expected packet size
+        // is under the MTU, but for now, since the # of packets acked is so few (usually
+        // just one or two), and since the packets are so small anyway, an additional five
+        // or ten bytes doesn't hurt.
+        if ( (ackIdsRemaining != null) && (ackIdsRemaining.size() > 0) )
+            data[off] |= UDPPacket.DATA_FLAG_EXPLICIT_ACK;
+        if ( (partialACKsRemaining != null) && (partialACKsRemaining.size() > 0) )
+            data[off] |= UDPPacket.DATA_FLAG_ACK_BITFIELDS;
         off++;
+
+        if ( (ackIdsRemaining != null) && (ackIdsRemaining.size() > 0) ) {
+            DataHelper.toLong(data, off, 1, ackIdsRemaining.size());
+            off++;
+            while (ackIdsRemaining.size() > 0) {
+                Long ackId = (Long)ackIdsRemaining.remove(0);
+                DataHelper.toLong(data, off, 4, ackId.longValue());
+                off += 4;
+            }
+        }
+
+        if ( (partialACKsRemaining != null) && (partialACKsRemaining.size() > 0) ) {
+            int origNumRemaining = partialACKsRemaining.size();
+            int numPartialOffset = off;
+            // leave it blank for now, since we could skip some
+            off++;
+            for (int i = 0; i < partialACKsRemaining.size(); i++) {
+                ACKBitfield bitfield = (ACKBitfield)partialACKsRemaining.get(i);
+                if (bitfield.receivedComplete()) continue;
+                DataHelper.toLong(data, off, 4, bitfield.getMessageId());
+                off += 4;
+                int bits = bitfield.fragmentCount();
+                int size = (bits / 7) + 1;
+                for (int curByte = 0; curByte < size; curByte++) {
+                    if (curByte + 1 < size)
+                        data[off] |= (byte)(1 << 7);
+                    
+                    for (int curBit = 0; curBit < 7; curBit++) {
+                        if (bitfield.received(curBit + 7*curByte))
+                            data[off] |= (byte)(1 << curBit);
+                    }
+                    off++;
+                }
+                partialACKsRemaining.remove(i);
+                i--;
+            }
+            // now jump back and fill in the number of bitfields *actually* included
+            DataHelper.toLong(data, numPartialOffset, 1, origNumRemaining - partialACKsRemaining.size());
+        }
         
         DataHelper.toLong(data, off, 1, 1); // only one fragment in this message
         off++;

router/java/src/net/i2p/router/transport/udp/PeerState.java

@@ -170,8 +170,8 @@ public class PeerState {
      * of 608
      */
     private static final int DEFAULT_MTU = 608;//600; //1500;
-    private static final int MIN_RTO = 800 + ACKSender.ACK_FREQUENCY;
-    private static final int MAX_RTO = 2000; // 5000;
+    private static final int MIN_RTO = 500 + ACKSender.ACK_FREQUENCY;
+    private static final int MAX_RTO = 5000; // 5000;
     
     public PeerState(I2PAppContext ctx) {
         _context = ctx;
@@ -515,7 +515,7 @@ public class PeerState {
         //if (true)
         //    _sendWindowBytes -= 10000;
         //else
-            _sendWindowBytes = _sendWindowBytes/4; //(_sendWindowBytes*2) / 3;
+            _sendWindowBytes = _sendWindowBytes/2; //(_sendWindowBytes*2) / 3;
         if (_sendWindowBytes < MINIMUM_WINDOW_BYTES)
             _sendWindowBytes = MINIMUM_WINDOW_BYTES;
         //if (congestionAt/2 < _slowStartThreshold)
@@ -523,6 +523,25 @@ public class PeerState {
         return true;
     }
     
+    /**
+     * Grab a list of message ids (Long) that we want to send to the remote
+     * peer, regardless of the packet size, but don't remove it from our 
+     * "want to send" list.  If the message id is transmitted to the peer,
+     * removeACKMessage(Long) should be called.
+     *
+     */
+    public List getCurrentFullACKs() {
+        synchronized (_currentACKs) {
+            return new ArrayList(_currentACKs);
+        }
+    }
+    public void removeACKMessage(Long messageId) {
+        synchronized (_currentACKs) {
+            _currentACKs.remove(messageId);
+        }
+        _lastACKSend = _context.clock().now();
+    }
+    
     /** 
      * grab a list of ACKBitfield instances, some of which may fully 
      * ACK a message while others may only partially ACK a message.  
@@ -578,7 +597,7 @@ public class PeerState {
         return rv;
     }
     
-    private void fetchPartialACKs(List rv) {
+    void fetchPartialACKs(List rv) {
         InboundMessageState states[] = null;
         int curState = 0;
         synchronized (_inboundMessages) {
@@ -632,7 +651,7 @@ public class PeerState {
                     float v = _context.random().nextFloat();
                     if (v < 0) v = 0-v;
                     if (v <= prob)
-                        _sendWindowBytes += 512; // bytesACKed;
+                        _sendWindowBytes += bytesACKed; //512; // bytesACKed;
                 }
             }
         }
@@ -640,7 +659,7 @@ public class PeerState {
             _sendWindowBytes = MAX_SEND_WINDOW_BYTES;
         _lastReceiveTime = _context.clock().now();
         
-        if (false) {
+        if (true) {
             if (_sendWindowBytesRemaining + bytesACKed <= _sendWindowBytes)
                 _sendWindowBytesRemaining += bytesACKed;
             else

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -890,17 +890,29 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             peers.addAll(_peersByIdent.values());
         }
         long offsetTotal = 0;
+
+        int bpsIn = 0;
+        int bpsOut = 0;
+        long uptimeMsTotal = 0;
+        long cwinTotal = 0;
+        long rttTotal = 0;
+        long rtoTotal = 0;
+        long sendTotal = 0;
+        long recvTotal = 0;
+        long resentTotal = 0;
+        long dupRecvTotal = 0;
+        int numPeers = 0;
         
         StringBuffer buf = new StringBuffer(512);
         buf.append("<b id=\"udpcon\">UDP connections: ").append(peers.size()).append("</b><br />\n");
         buf.append("<table border=\"1\">\n");
-        buf.append(" <tr><td><b>peer</b></td><td><b>idle</b></td>");
-        buf.append("     <td><b>in/out</b></td>\n");
-        buf.append("     <td><b>up</b></td><td><b>skew</b></td>\n");
-        buf.append("     <td><b>cwnd</b></td><td><b>ssthresh</b></td>\n");
-        buf.append("     <td><b>rtt</b></td><td><b>dev</b></td><td><b>rto</b></td>\n");
-        buf.append("     <td><b>send</b></td><td><b>recv</b></td>\n");
-        buf.append("     <td><b>resent</b></td><td><b>dupRecv</b></td>\n");
+        buf.append(" <tr><td><b><a href=\"#def.peer\">peer</a></b></td><td><b><a href=\"#def.idle\">idle</a></b></td>");
+        buf.append("     <td><b><a href=\"#def.rate\">in/out</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.up\">up</a></b></td><td><b><a href=\"#def.skew\">skew</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.cwnd\">cwnd</a></b></td><td><b><a href=\"#def.ssthresh\">ssthresh</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.rtt\">rtt</a></b></td><td><b><a href=\"#def.dev\">dev</a></b></td><td><b><a href=\"#def.rto\">rto</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.send\">send</a></b></td><td><b><a href=\"#def.recv\">recv</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.resent\">resent</a></b></td><td><b><a href=\"#def.dupRecv\">dupRecv</a></b></td>\n");
         buf.append(" </tr>\n");
         out.write(buf.toString());
         buf.setLength(0);
@@ -968,16 +980,22 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             }
             buf.append("</code></td>");
             
+            long idleIn = (now-peer.getLastReceiveTime())/1000;
+            long idleOut = (now-peer.getLastSendTime())/1000;
+            
             buf.append("<td valign=\"top\" ><code>");
-            buf.append((now-peer.getLastReceiveTime())/1000);
+            buf.append(idleIn);
             buf.append("s/");
-            buf.append((now-peer.getLastSendTime())/1000);
+            buf.append(idleOut);
             buf.append("s</code></td>");
-    
+ 
+            int recvBps = (idleIn > 10 ? 0 : peer.getReceiveBps());
+            int sendBps = (idleOut > 10 ? 0 : peer.getSendBps());
+            
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(formatKBps(peer.getReceiveBps()));
+            buf.append(formatKBps(recvBps));
             buf.append("KBps/");
-            buf.append(formatKBps(peer.getSendBps()));
+            buf.append(formatKBps(sendBps));
             buf.append("KBps ");
             //buf.append(formatKBps(peer.getReceiveACKBps()));
             //buf.append("KBps/");
@@ -985,8 +1003,10 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             //buf.append("KBps ");
             buf.append("</code></td>");
 
+            long uptime = now - peer.getKeyEstablishedTime();
+            
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(DataHelper.formatDuration(now-peer.getKeyEstablishedTime()));
+            buf.append(DataHelper.formatDuration(uptime));
             buf.append("</code></td>");
             
             buf.append("<td valign=\"top\" ><code>");
@@ -994,16 +1014,21 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             buf.append("s</code></td>");
             offsetTotal = offsetTotal + peer.getClockSkew();
 
+            long sendWindow = peer.getSendWindowBytes();
+            
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getSendWindowBytes()/1024);
+            buf.append(sendWindow/1024);
             buf.append("K</code></td>");
 
             buf.append("<td valign=\"top\" ><code>");
             buf.append(peer.getSlowStartThreshold()/1024);
             buf.append("K</code></td>");
 
+            int rtt = peer.getRTT();
+            int rto = peer.getRTO();
+            
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getRTT());
+            buf.append(rtt);
             buf.append("</code></td>");
             
             buf.append("<td valign=\"top\" ><code>");
@@ -1011,48 +1036,92 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             buf.append("</code></td>");
 
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getRTO());
+            buf.append(rto);
+            buf.append("</code></td>");
+        
+            long sent = peer.getPacketsTransmitted();
+            long recv = peer.getPacketsReceived();
+            
+            buf.append("<td valign=\"top\" ><code>");
+            buf.append(sent);
             buf.append("</code></td>");
             
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getPacketsTransmitted());
+            buf.append(recv);
             buf.append("</code></td>");
             
-            buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getPacketsReceived());
-            buf.append("</code></td>");
+            //double sent = (double)peer.getPacketsPeriodTransmitted();
+            //double sendLostPct = 0;
+            //if (sent > 0)
+            //    sendLostPct = (double)peer.getPacketsRetransmitted()/(sent);
             
-            double sent = (double)peer.getPacketsPeriodTransmitted();
-            double sendLostPct = 0;
-            if (sent > 0)
-                sendLostPct = (double)peer.getPacketsRetransmitted()/(sent);
+            long resent = peer.getPacketsRetransmitted();
+            long dupRecv = peer.getPacketsReceivedDuplicate();
             
             buf.append("<td valign=\"top\" ><code>");
             //buf.append(formatPct(sendLostPct));
-            buf.append(peer.getPacketsRetransmitted()); // + "/" + peer.getPacketsPeriodRetransmitted() + "/" + sent);
+            buf.append(resent); // + "/" + peer.getPacketsPeriodRetransmitted() + "/" + sent);
             //buf.append(peer.getPacketRetransmissionRate());
             buf.append("</code></td>");
             
             double recvDupPct = (double)peer.getPacketsReceivedDuplicate()/(double)peer.getPacketsReceived();
             buf.append("<td valign=\"top\" ><code>");
-            buf.append(peer.getPacketsReceivedDuplicate()); //formatPct(recvDupPct));
+            buf.append(dupRecv); //formatPct(recvDupPct));
             buf.append("</code></td>");
 
             buf.append("</tr>");
             out.write(buf.toString());
             buf.setLength(0);
+            
+            bpsIn += recvBps;
+            bpsOut += sendBps;
+        
+            uptimeMsTotal += uptime;
+            cwinTotal += sendWindow;
+            rttTotal += rtt;
+            rtoTotal += rto;
+        
+            sendTotal += sent;
+            recvTotal += recv;
+            resentTotal += resent;
+            dupRecvTotal += dupRecv;
+            
+            numPeers++;
         }
         
-        out.write("</table>\n");
-        
-        buf.append("<b>Average clock skew, UDP peers:");
-        if (peers.size() > 0)
-            buf.append(offsetTotal / peers.size()).append("ms</b><br><br>\n");
-        else
-            buf.append("n/a</b><br><br>\n");
-        
+        buf.append("<tr><td colspan=\"14\"><hr /></td></tr>\n");
+        buf.append(" <tr><td colspan=\"2\"><b>Total</b></td>");
+        buf.append("     <td>");
+        buf.append(formatKBps(bpsIn)).append("KBps/").append(formatKBps(bpsOut));
+        buf.append("KBps</td>");
+        buf.append("     <td>").append(numPeers > 0 ? DataHelper.formatDuration(uptimeMsTotal/numPeers) : "0s");
+        buf.append("</td><td>").append(numPeers > 0 ? DataHelper.formatDuration(offsetTotal/numPeers) : "0ms").append("</td>\n");
+        buf.append("     <td>");
+        buf.append(numPeers > 0 ? cwinTotal/(numPeers*1024) + "K" : "0K");
+        buf.append("</td><td>&nbsp;</td>\n");
+        buf.append("     <td>");
+        buf.append(numPeers > 0 ? rttTotal/numPeers : 0);
+        buf.append("</td><td>&nbsp;</td><td>");
+        buf.append(numPeers > 0 ? rtoTotal/numPeers : 0);
+        buf.append("</td>\n     <td>");
+        buf.append(sendTotal).append("</td><td>").append(recvTotal).append("</td>\n");
+        buf.append("     <td>").append(resentTotal);
+        buf.append("</td><td>").append(dupRecvTotal).append("</td>\n");
+        buf.append(" </tr>\n");
+        buf.append("<tr><td colspan=\"14\" valign=\"top\" align=\"left\">");
+        long bytesTransmitted = _context.bandwidthLimiter().getTotalAllocatedOutboundBytes();
+        double averagePacketSize = _context.statManager().getRate("udp.sendPacketSize").getLifetimeAverageValue();
+        // lifetime value, not just the retransmitted packets of current connections
+        resentTotal = (long)_context.statManager().getRate("udp.packetsRetransmitted").getLifetimeEventCount();
+        double nondupSent = ((double)bytesTransmitted - ((double)resentTotal)*averagePacketSize);
+        double bwResent = (nondupSent <= 0 ? 0d : ((((double)resentTotal)*averagePacketSize) / nondupSent));
+        buf.append("Percentage of bytes retransmitted (lifetime): ").append(formatPct(bwResent));
+        buf.append(" <i>(includes retransmission required by packet loss)</i><br />\n");
+        buf.append("</td></tr>\n");
         out.write(buf.toString());
         buf.setLength(0);
+        out.write(KEY);
+        out.write("</table>\n");
     }
     
     private static final DecimalFormat _fmt = new DecimalFormat("#,##0.00");
@@ -1068,6 +1137,23 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         }
     }
     
+    private static final String KEY = "<tr><td colspan=\"14\" valign=\"top\" align=\"left\">" +
+        "<b id=\"def.peer\">peer</b>: the remote peer (&lt; means they offer to introduce us, &gt; means we offer to introduce them)<br />\n" +
+        "<b id=\"def.idle\">idle</b>: the idle time is how long since a packet has been received or sent<br />\n" +
+        "<b id=\"def.rate\">in/out</b>: the rates show a smoothed inbound and outbound transfer rate (KBytes per second)<br />\n" +
+        "<b id=\"def.up\">up</b>: the uptime is how long ago this session was established<br />\n" +
+        "<b id=\"def.skew\">skew</b>: the skew says how far off the other user's clock is, relative to your own<br />\n" +
+        "<b id=\"def.cwnd\">cwnd</b>: the congestion window is how many bytes in 'in flight' you can send without an acknowledgement<br />\n" +
+        "<b id=\"def.ssthresh\">ssthresh</b>: the slow start threshold help make sure the cwnd doesn't grow too fast<br />\n" +
+        "<b id=\"def.rtt\">rtt</b>: the round trip time is how long it takes to get an acknowledgement of a packet<br />\n" +
+        "<b id=\"def.dev\">dev</b>: the standard deviation of the round trip time, to help control the retransmit timeout<br />\n" +
+        "<b id=\"def.rto\">rto</b>: the retransmit timeout controls how frequently an unacknowledged packet will be retransmitted<br />\n" +
+        "<b id=\"def.send\">send</b>: the number of packets sent to the peer<br />\n" +
+        "<b id=\"def.recv\">recv</b>: the number of packets received from the peer<br />\n" +
+        "<b id=\"def.resent\">resent</b>: the number of packets retransmitted to the peer<br />\n" +
+        "<b id=\"def.dupRecv\">dupRecv</b>: the number of duplicate packets received from the peer" +
+        "</td></tr>\n";
+    
     /**
      * Cache the bid to reduce object churn
      */
@@ -1278,4 +1364,23 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             }
         }
     }
+    
+    private static final String BADIPS[] = new String[] { "192.168.0.1", "127.0.0.1", "10.3.4.5", "172.16.3.4", "224.5.6.7" };
+    private static final String GOODIPS[] = new String[] { "192.167.0.1", "126.0.0.1", "11.3.4.5", "172.15.3.4", "223.5.6.7" };
+    public static void main(String args[]) {
+        for (int i = 0; i < BADIPS.length; i++) {
+            try { 
+                InetAddress addr = InetAddress.getByName(BADIPS[i]);
+                boolean routable = isPubliclyRoutable(addr.getAddress());
+                System.out.println("Routable: " + routable + " (" + BADIPS[i] + ")");
+            } catch (Exception e) { e.printStackTrace(); }
+        }
+        for (int i = 0; i < GOODIPS.length; i++) {
+            try { 
+                InetAddress addr = InetAddress.getByName(GOODIPS[i]);
+                boolean routable = isPubliclyRoutable(addr.getAddress());
+                System.out.println("Routable: " + routable + " (" + GOODIPS[i] + ")");
+            } catch (Exception e) { e.printStackTrace(); }
+        }
+    }
 }

router/java/src/net/i2p/router/tunnel/pool/RequestTunnelJob.java

@@ -65,10 +65,12 @@ public class RequestTunnelJob extends JobImpl {
         ctx.statManager().createRateStat("tunnel.receiveRejectionTransient", "How often we are rejected due to transient overload?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         ctx.statManager().createRateStat("tunnel.receiveRejectionBandwidth", "How often we are rejected due to bandwidth overload?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         ctx.statManager().createRateStat("tunnel.receiveRejectionCritical", "How often we are rejected due to critical failure?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
-        ctx.statManager().createRateStat("tunnel.buildFailure", "How often we fail to build a non-exploratory tunnel?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
-        ctx.statManager().createRateStat("tunnel.buildExploratoryFailure", "How often we fail to build an exploratory tunnel?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
+        ctx.statManager().createRateStat("tunnel.buildFailure", "What hop was being requested when a nonexploratory tunnel request failed?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
+        ctx.statManager().createRateStat("tunnel.buildExploratoryFailure", "What hop was beiing requested when an exploratory tunnel request failed?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         ctx.statManager().createRateStat("tunnel.buildSuccess", "How often we succeed building a non-exploratory tunnel?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         ctx.statManager().createRateStat("tunnel.buildExploratorySuccess", "How often we succeed building an exploratory tunnel?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
+        ctx.statManager().createRateStat("tunnel.buildPartialTime", "How long a non-exploratory request took to be accepted?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
+        ctx.statManager().createRateStat("tunnel.buildExploratoryPartialTime", "How long an exploratory request took to be accepted?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
 
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Requesting hop " + hop + " in " + cfg);
@@ -262,14 +264,20 @@ public class RequestTunnelJob extends JobImpl {
         if (_onFailed != null)
             getContext().jobQueue().addJob(_onFailed);
         if (_isExploratory)
-            getContext().statManager().addRateData("tunnel.buildExploratoryFailure", 1, 0);
+            getContext().statManager().addRateData("tunnel.buildExploratoryFailure", _currentHop, _config.getLength());
         else
-            getContext().statManager().addRateData("tunnel.buildFailure", 1, 0);
+            getContext().statManager().addRateData("tunnel.buildFailure", _currentHop, _config.getLength());
     }
     
     private void peerSuccess() {
+        long now = getContext().clock().now();
         getContext().profileManager().tunnelJoined(_currentPeer.getIdentity().calculateHash(), 
-                                                   getContext().clock().now() - _lastSendTime);
+                                                   now - _lastSendTime);
+        if (_isExploratory)
+            getContext().statManager().addRateData("tunnel.buildExploratoryPartialTime", now - _lastSendTime, 0);
+        else
+            getContext().statManager().addRateData("tunnel.buildPartialTime", now - _lastSendTime, 0);
+
         if (_currentHop > 0) {
             RequestTunnelJob j = new RequestTunnelJob(getContext(), _config, _onCreated, _onFailed, _currentHop - 1, _isFake, _isExploratory);
             getContext().jobQueue().addJob(j);