Release 0.6.14

Accordion

See merge request zlatinb/muwire!41

.gitignore

@@ -2,7 +2,7 @@
 **/.settings
 **/build
 .gradle
-.project
-.classpath
+**/.project
+**/.classpath
 **/*.rej
 **/*.orig

README.md

@@ -1,3 +1,5 @@
+The GitHub repo is mirrored from the in-I2P GitLab repo.  Please open PRs and issues at http://git.idk.i2p/zlatinb/muwire
+
 # MuWire - Easy Anonymous File-Sharing
 
 MuWire is an easy to use file-sharing program which offers anonymity using [I2P technology](http://geti2p.net).  It works on any platform Java works on, including Windows,MacOS,Linux.
@@ -49,6 +51,9 @@ MuWire is available as a Docker image.  For more information see the [Docker] pa
 ## Translations
 If you want to help translate MuWire, instructions are on the wiki https://github.com/zlatinb/muwire/wiki/Translate
 
+## MuWire Tracker Daemon
+The MuWire Tracker Daemon (or mwtrackerd for short) is a project to bring functionality similar to BitTorrent tracking to MuWire.  For more info see the [Tracker] page.
+
 ## GPG Fingerprint
 
 ```
@@ -67,3 +72,4 @@ You can find the full key at https://keybase.io/zlatinb
 [Plugin]: https://github.com/zlatinb/muwire/wiki/Plugin
 [Docker]: https://github.com/zlatinb/muwire/wiki/Docker
 [jlesage/docker-baseimage-gui]: https://github.com/jlesage/docker-baseimage-gui
+[Tracker]: https://github.com/zlatinb/muwire/wiki/Tracker-Daemon

TODO.md

@@ -19,7 +19,8 @@ This helps with scalability
 * Enum i18n
 * Ability to share trust list only with trusted users
 * Confidential files visible only to certain users
-* Public Feed feature
+* Advertise file feed and browseability in upload headers
+* Manual polling / shared folder re-scan (because polling NAS doesn't work)
 
 ### Chat
 * echo "unknown/innappropriate command" in the console
@@ -32,11 +33,14 @@ This helps with scalability
 ### Swing GUI
 * I2P Status panel - display message when connected to external router
 * Search box - left identation
+* Ability to disable switching of tabs on actions
+* Ability to trust/browse/subscribe from uploads tab
 
 ### Web UI/Plugin
 * HTML 5 media players
-* Minimal dependency (break up groovy-all.jar)
 * Remove versions from jar names
 * Security: POST nonces, CSP headers
+* Upload files from browser to plugin via drag-and-drop
+* Check permissions, display better errors when sharing local folders
 
  

build.gradle

@@ -9,6 +9,19 @@ subprojects {
 
     compileGroovy {
         groovyOptions.optimizationOptions.indy = true
+        sourceCompatibility = project.sourceCompatibility
+        targetCompatibility = project.targetCompatibility
+        options.compilerArgs += project.compilerArgs
+        options.deprecation = true
+        options.encoding = 'UTF-8'
+    }
+
+    compileJava {
+        sourceCompatibility = project.sourceCompatibility
+        targetCompatibility = project.targetCompatibility
+        options.compilerArgs += project.compilerArgs
+        options.deprecation = true
+        options.encoding = 'UTF-8'
     }
 
     repositories {

cli-lanterna/src/main/groovy/com/muwire/clilanterna/CliLanterna.groovy

@@ -32,7 +32,7 @@ import com.muwire.core.UILoadedEvent
 import com.muwire.core.files.AllFilesLoadedEvent
 
 class CliLanterna {
-    private static final String MW_VERSION = "0.6.10"
+    private static final String MW_VERSION = "0.6.14"
     
     private static volatile Core core
     

cli-lanterna/src/main/groovy/com/muwire/clilanterna/FilesModel.groovy

@@ -28,7 +28,6 @@ class FilesModel {
         core.eventBus.register(FileLoadedEvent.class, this)
         core.eventBus.register(FileUnsharedEvent.class, this)
         core.eventBus.register(FileHashedEvent.class, this)
-        core.eventBus.register(AllFilesLoadedEvent.class, this)
         
         Runnable refreshModel = {refreshModel()}
         Timer timer = new Timer(true)
@@ -38,15 +37,6 @@ class FilesModel {
         
     }
     
-    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
-        def eventBus = core.eventBus
-        guiThread.invokeLater {
-            core.muOptions.watchedDirectories.each {
-                eventBus.publish(new FileSharedEvent(file: new File(it)))
-            }
-        }
-    }
-    
     void onFileLoadedEvent(FileLoadedEvent e) {
         guiThread.invokeLater {
             sharedFiles.add(e.loadedFile)

core/build.gradle

@@ -1,12 +1,38 @@
-apply plugin : 'application'
-mainClassName = 'com.muwire.core.Core'
-applicationDefaultJvmArgs = ['-Djava.util.logging.config.file=logging.properties']
-dependencies {
-    compile "net.i2p:i2p:${i2pVersion}"
-    compile "net.i2p:router:${i2pVersion}"
-    compile "net.i2p.client:mstreaming:${i2pVersion}"
-    compile "net.i2p.client:streaming:${i2pVersion}"
 
-    testCompile 'org.junit.jupiter:junit-jupiter-api:5.4.2'    
-    testCompile 'junit:junit:4.12'
+plugins {
+    id 'java-library'
+    id 'maven-publish'
+}
+
+dependencies {
+    api "net.i2p:i2p:${i2pVersion}"
+    api "net.i2p:router:${i2pVersion}"
+    implementation "net.i2p.client:mstreaming:${i2pVersion}"
+    implementation "net.i2p.client:streaming:${i2pVersion}"
+
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2'    
+    testImplementation 'junit:junit:4.12'
+    testImplementation 'org.codehaus.groovy:groovy-all:2.4.15'
+}
+
+
+// this is necessary because applying both groovy and java-library doesn't work well
+configurations {
+    apiElements.outgoing.variants {
+        classes {
+            artifact file: compileGroovy.destinationDir, builtBy: compileGroovy
+        }
+    }
+}
+
+// publish core to local maven repo for sister projects
+publishing {
+    publications {
+        muCore(MavenPublication) {
+            from components.java
+        }
+    }
+    repositories {
+        mavenLocal()
+    }
 }

core/src/main/groovy/com/muwire/core/Core.groovy

@@ -5,6 +5,8 @@ import com.muwire.core.files.PersisterFolderService
 
 import java.nio.charset.StandardCharsets
 import java.util.concurrent.atomic.AtomicBoolean
+import java.util.logging.Level
+import java.util.zip.ZipException
 
 import com.muwire.core.chat.ChatDisconnectionEvent
 import com.muwire.core.chat.ChatManager
@@ -32,6 +34,16 @@ import com.muwire.core.filecert.CertificateManager
 import com.muwire.core.filecert.UICreateCertificateEvent
 import com.muwire.core.filecert.UIFetchCertificatesEvent
 import com.muwire.core.filecert.UIImportCertificateEvent
+import com.muwire.core.filefeeds.FeedClient
+import com.muwire.core.filefeeds.FeedFetchEvent
+import com.muwire.core.filefeeds.FeedItemFetchedEvent
+import com.muwire.core.filefeeds.FeedManager
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
+import com.muwire.core.filefeeds.UIFeedDeletedEvent
+import com.muwire.core.filefeeds.UIFeedUpdateEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
 import com.muwire.core.files.FileDownloadedEvent
 import com.muwire.core.files.FileHashedEvent
 import com.muwire.core.files.FileHasher
@@ -43,7 +55,11 @@ import com.muwire.core.files.HasherService
 import com.muwire.core.files.PersisterService
 import com.muwire.core.files.SideCarFileEvent
 import com.muwire.core.files.UICommentEvent
-
+import com.muwire.core.files.directories.UISyncDirectoryEvent
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectoryConvertedEvent
+import com.muwire.core.files.directories.WatchedDirectoryConverter
+import com.muwire.core.files.directories.WatchedDirectoryManager
 import com.muwire.core.files.AllFilesLoadedEvent
 import com.muwire.core.files.DirectoryUnsharedEvent
 import com.muwire.core.files.DirectoryWatchedEvent
@@ -69,6 +85,7 @@ import com.muwire.core.upload.UploadManager
 import com.muwire.core.util.MuWireLogManager
 import com.muwire.core.content.ContentControlEvent
 import com.muwire.core.content.ContentManager
+import com.muwire.core.tracker.TrackerResponder
 
 import groovy.util.logging.Log
 import net.i2p.I2PAppContext
@@ -96,19 +113,19 @@ public class Core {
     final Properties i2pOptions
     final MuWireSettings muOptions
 
-    private final I2PSession i2pSession;
+    final I2PSession i2pSession;
     final TrustService trustService
     final TrustSubscriber trustSubscriber
     private final PersisterService persisterService
     private final PersisterFolderService persisterFolderService
-    private final HostCache hostCache
-    private final ConnectionManager connectionManager
+    final HostCache hostCache
+    final ConnectionManager connectionManager
     private final CacheClient cacheClient
     private final UpdateClient updateClient
-    private final ConnectionAcceptor connectionAcceptor
+    final ConnectionAcceptor connectionAcceptor
     private final ConnectionEstablisher connectionEstablisher
     private final HasherService hasherService
-    private final DownloadManager downloadManager
+    final DownloadManager downloadManager
     private final DirectoryWatcher directoryWatcher
     final FileManager fileManager
     final UploadManager uploadManager
@@ -116,6 +133,11 @@ public class Core {
     final CertificateManager certificateManager
     final ChatServer chatServer
     final ChatManager chatManager
+    final FeedManager feedManager
+    private final FeedClient feedClient
+    private final WatchedDirectoryConverter watchedDirectoryConverter
+    final WatchedDirectoryManager watchedDirectoryManager
+    private final TrackerResponder trackerResponder
 
     private final Router router
 
@@ -132,19 +154,23 @@ public class Core {
         // Read defaults
         def defaultI2PFile = getClass()
                 .getClassLoader().getResource("defaults/i2p.properties");
-        defaultI2PFile.withInputStream { i2pOptions.load(it) }
+        try {
+            defaultI2PFile.withInputStream { i2pOptions.load(it) }
+        } catch (ZipException mystery) {
+            log.log(Level.SEVERE, "couldn't load default i2p properties", mystery)
+        }
 
         def i2pOptionsFile = new File(home, "i2p.properties")
         if (i2pOptionsFile.exists()) {
             i2pOptionsFile.withInputStream { i2pOptions.load(it) }
 
             if (!i2pOptions.containsKey("inbound.nickname"))
-                i2pOptions["inbound.nickname"] = "MuWire"
+                i2pOptions["inbound.nickname"] = tunnelName
             if (!i2pOptions.containsKey("outbound.nickname"))
-                i2pOptions["outbound.nickname"] = "MuWire"
+                i2pOptions["outbound.nickname"] = tunnelName
         }
-        if (!(i2pOptions.hasProperty("i2np.ntcp.port")
-                && i2pOptions.hasProperty("i2np.udp.port")
+        if (!(i2pOptions.containsKey("i2np.ntcp.port")
+                && i2pOptions.containsKey("i2np.udp.port")
         )) {
             Random r = new Random()
             int port = r.nextInt(60000) + 4000
@@ -194,7 +220,7 @@ public class Core {
         // options like tunnel length and quantity
         I2PSocketManager socketManager
         keyDat.withInputStream {
-            socketManager = new I2PSocketManagerFactory().createManager(it, i2pOptions["i2cp.tcp.host"], i2pOptions["i2cp.tcp.port"].toInteger(), i2pOptions)
+            socketManager = new I2PSocketManagerFactory().createDisconnectedManager(it, i2pOptions["i2cp.tcp.host"], i2pOptions["i2cp.tcp.port"].toInteger(), i2pOptions)
         }
         socketManager.getDefaultOptions().setReadTimeout(60000)
         socketManager.getDefaultOptions().setConnectTimeout(30000)
@@ -269,6 +295,8 @@ public class Core {
         eventBus.register(FileHashedEvent.class, persisterFolderService)
         eventBus.register(FileUnsharedEvent.class, persisterFolderService)
         eventBus.register(UICommentEvent.class, persisterFolderService)
+        eventBus.register(UIFilePublishedEvent.class, persisterFolderService)
+        eventBus.register(UIFileUnpublishedEvent.class, persisterFolderService)
 
         log.info("initializing host cache")
         File hostStorage = new File(home, "hosts.json")
@@ -311,6 +339,19 @@ public class Core {
             register(TrustEvent.class, chatServer)
         }
         
+        log.info("initializing feed manager")
+        feedManager = new FeedManager(eventBus, home)
+        eventBus.with { 
+            register(FeedItemFetchedEvent.class, feedManager)
+            register(FeedFetchEvent.class, feedManager)
+            register(UIFeedConfigurationEvent.class, feedManager)
+            register(UIFeedDeletedEvent.class, feedManager)
+        }
+        
+        log.info("initializing feed client")
+        feedClient = new FeedClient(i2pConnector, eventBus, me, feedManager)
+        eventBus.register(UIFeedUpdateEvent.class, feedClient)
+        
         log.info "initializing results sender"
         ResultsSender resultsSender = new ResultsSender(eventBus, i2pConnector, me, props, certificateManager, chatServer)
 
@@ -322,6 +363,7 @@ public class Core {
         log.info("initializing download manager")
         downloadManager = new DownloadManager(eventBus, trustService, meshManager, props, i2pConnector, home, me)
         eventBus.register(UIDownloadEvent.class, downloadManager)
+        eventBus.register(UIDownloadFeedItemEvent.class, downloadManager)
         eventBus.register(UILoadedEvent.class, downloadManager)
         eventBus.register(FileDownloadedEvent.class, downloadManager)
         eventBus.register(UIDownloadCancelledEvent.class, downloadManager)
@@ -331,6 +373,9 @@ public class Core {
 
         log.info("initializing upload manager")
         uploadManager = new UploadManager(eventBus, fileManager, meshManager, downloadManager, persisterFolderService, props)
+        
+        log.info("initializing tracker responder")
+        trackerResponder = new TrackerResponder(i2pSession, props, fileManager, downloadManager, meshManager, trustService, me)
 
         log.info("initializing connection establisher")
         connectionEstablisher = new ConnectionEstablisher(eventBus, i2pConnector, props, connectionManager, hostCache)
@@ -351,11 +396,6 @@ public class Core {
             i2pAcceptor, hostCache, trustService, searchManager, uploadManager, fileManager, connectionEstablisher,
             certificateManager, chatServer)
 
-        log.info("initializing directory watcher")
-        directoryWatcher = new DirectoryWatcher(eventBus, fileManager, home, props)
-        eventBus.register(DirectoryWatchedEvent.class, directoryWatcher)
-        eventBus.register(AllFilesLoadedEvent.class, directoryWatcher)
-        eventBus.register(DirectoryUnsharedEvent.class, directoryWatcher)
 
         log.info("initializing hasher service")
         hasherService = new HasherService(new FileHasher(), eventBus, fileManager, props)
@@ -377,9 +417,32 @@ public class Core {
         BrowseManager browseManager = new BrowseManager(i2pConnector, eventBus, me)
         eventBus.register(UIBrowseEvent.class, browseManager)
         
+        log.info("initializing watched directory converter")
+        watchedDirectoryConverter = new WatchedDirectoryConverter(this)
+        eventBus.register(AllFilesLoadedEvent.class, watchedDirectoryConverter)
+        
+        log.info("initializing watched directory manager")
+        watchedDirectoryManager = new WatchedDirectoryManager(home, eventBus, fileManager)
+        eventBus.with { 
+            register(WatchedDirectoryConfigurationEvent.class, watchedDirectoryManager)
+            register(WatchedDirectoryConvertedEvent.class, watchedDirectoryManager)
+            register(FileSharedEvent.class, watchedDirectoryManager)
+            register(DirectoryUnsharedEvent.class, watchedDirectoryManager)
+            register(UISyncDirectoryEvent.class, watchedDirectoryManager)
+        }
+        
+        log.info("initializing directory watcher")
+        directoryWatcher = new DirectoryWatcher(eventBus, fileManager, home, watchedDirectoryManager)
+        eventBus.with {
+            register(DirectoryWatchedEvent.class, directoryWatcher)
+            register(WatchedDirectoryConvertedEvent.class, directoryWatcher)
+            register(DirectoryUnsharedEvent.class, directoryWatcher)
+            register(WatchedDirectoryConfigurationEvent.class, directoryWatcher)
+        }
     }
 
     public void startServices() {
+        i2pSession.connect()
         hasherService.start()
         trustService.start()
         trustService.waitForLoad()
@@ -390,6 +453,9 @@ public class Core {
         connectionEstablisher.start()
         hostCache.waitForLoad()
         updateClient?.start()
+        feedManager.start()
+        feedClient.start()
+        trackerResponder.start()
     }
 
     public void shutdown() {
@@ -417,12 +483,20 @@ public class Core {
         connectionEstablisher.stop()
         log.info("shutting down directory watcher")
         directoryWatcher.stop()
+        log.info("shutting down watch directory manager")
+        watchedDirectoryManager.shutdown()
         log.info("shutting down cache client")
         cacheClient.stop()
         log.info("shutting down chat server")
         chatServer.stop()
         log.info("shutting down chat manager")
         chatManager.shutdown()
+        log.info("shutting down feed manager")
+        feedManager.stop()
+        log.info("shutting down feed client")
+        feedClient.stop()
+        log.info("shutting down tracker responder")
+        trackerResponder.stop()
         log.info("shutting down connection manager")
         connectionManager.shutdown()
         log.info("killing i2p session")
@@ -470,7 +544,7 @@ public class Core {
             }
         }
 
-        Core core = new Core(props, home, "0.6.10")
+        Core core = new Core(props, home, "0.6.14")
         core.startServices()
 
         // ... at the end, sleep or execute script

core/src/main/groovy/com/muwire/core/MuWireSettings.groovy

@@ -31,6 +31,17 @@ class MuWireSettings {
     boolean shareHiddenFiles
     boolean searchComments
     boolean browseFiles
+    boolean allowTracking
+    
+    boolean fileFeed
+    boolean advertiseFeed
+    boolean autoPublishSharedFiles
+    boolean defaultFeedAutoDownload
+    int defaultFeedUpdateInterval
+    int defaultFeedItemsToKeep
+    boolean defaultFeedSequential
+    
+    
     boolean startChatServer
     int maxChatConnections
     boolean advertiseChat
@@ -82,6 +93,17 @@ class MuWireSettings {
         outBw = Integer.valueOf(props.getProperty("outBw","128"))
         searchComments = Boolean.valueOf(props.getProperty("searchComments","true"))
         browseFiles = Boolean.valueOf(props.getProperty("browseFiles","true"))
+        allowTracking = Boolean.valueOf(props.getProperty("allowTracking","true"))
+        
+        // feed settings
+        fileFeed = Boolean.valueOf(props.getProperty("fileFeed","true"))
+        advertiseFeed = Boolean.valueOf(props.getProperty("advertiseFeed","true"))
+        autoPublishSharedFiles = Boolean.valueOf(props.getProperty("autoPublishSharedFiles", "false"))
+        defaultFeedAutoDownload = Boolean.valueOf(props.getProperty("defaultFeedAutoDownload", "false"))
+        defaultFeedItemsToKeep = Integer.valueOf(props.getProperty("defaultFeedItemsToKeep", "1000"))
+        defaultFeedSequential = Boolean.valueOf(props.getProperty("defaultFeedSequential", "false"))
+        defaultFeedUpdateInterval = Integer.valueOf(props.getProperty("defaultFeedUpdateInterval", "60000"))
+        
         speedSmoothSeconds = Integer.valueOf(props.getProperty("speedSmoothSeconds","60"))
         totalUploadSlots = Integer.valueOf(props.getProperty("totalUploadSlots","-1"))
         uploadSlotsPerUser = Integer.valueOf(props.getProperty("uploadSlotsPerUser","-1"))
@@ -137,6 +159,17 @@ class MuWireSettings {
         props.setProperty("outBw", String.valueOf(outBw))
         props.setProperty("searchComments", String.valueOf(searchComments))
         props.setProperty("browseFiles", String.valueOf(browseFiles))
+        props.setProperty("allowTracking", String.valueOf(allowTracking))
+        
+        // feed settings
+        props.setProperty("fileFeed", String.valueOf(fileFeed))
+        props.setProperty("advertiseFeed", String.valueOf(advertiseFeed))
+        props.setProperty("autoPublishSharedFiles", String.valueOf(autoPublishSharedFiles))
+        props.setProperty("defaultFeedAutoDownload", String.valueOf(defaultFeedAutoDownload))
+        props.setProperty("defaultFeedItemsToKeep", String.valueOf(defaultFeedItemsToKeep))
+        props.setProperty("defaultFeedSequential", String.valueOf(defaultFeedSequential))
+        props.setProperty("defaultFeedUpdateInterval", String.valueOf(defaultFeedUpdateInterval))
+        
         props.setProperty("speedSmoothSeconds", String.valueOf(speedSmoothSeconds))
         props.setProperty("totalUploadSlots", String.valueOf(totalUploadSlots))
         props.setProperty("uploadSlotsPerUser", String.valueOf(uploadSlotsPerUser))

core/src/main/groovy/com/muwire/core/connection/ConnectionAcceptor.groovy

@@ -15,9 +15,11 @@ import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
 import com.muwire.core.Persona
+import com.muwire.core.SharedFile
 import com.muwire.core.chat.ChatServer
 import com.muwire.core.filecert.Certificate
 import com.muwire.core.filecert.CertificateManager
+import com.muwire.core.filefeeds.FeedItems
 import com.muwire.core.files.FileManager
 import com.muwire.core.hostcache.HostCache
 import com.muwire.core.trust.TrustLevel
@@ -58,7 +60,7 @@ class ConnectionAcceptor {
 
     private volatile shutdown
     
-    private volatile int browsed
+    volatile int browsed
 
     ConnectionAcceptor(EventBus eventBus, UltrapeerConnectionManager manager,
         MuWireSettings settings, I2PAcceptor acceptor, HostCache hostCache,
@@ -161,6 +163,9 @@ class ConnectionAcceptor {
                 case (byte)'I':
                     processIRC(e)
                     break
+                case (byte)'F':
+                    processFEED(e)
+                    break
                 default:
                     throw new Exception("Invalid read $read")
             }
@@ -310,6 +315,9 @@ class ConnectionAcceptor {
             boolean chat = false
             if (headers.containsKey('Chat'))
                 chat = Boolean.parseBoolean(headers['Chat'])
+            boolean feed = false
+            if (headers.containsKey('Feed'))
+                feed = Boolean.parseBoolean(headers['Feed'])
                 
             byte [] personaBytes = Base64.decode(headers['Sender'])
             Persona sender = new Persona(new ByteArrayInputStream(personaBytes))
@@ -329,6 +337,7 @@ class ConnectionAcceptor {
                 def json = slurper.parse(payload)
                 results[i] = ResultsParser.parse(sender, resultsUUID, json)
                 results[i].chat = chat
+                results[i].feed = feed
             }
             eventBus.publish(new UIResultBatchEvent(uuid: resultsUUID, results: results))
         } catch (IOException bad) {
@@ -374,6 +383,9 @@ class ConnectionAcceptor {
             boolean chat = chatServer.running.get() && settings.advertiseChat
             os.write("Chat: ${chat}\r\n".getBytes(StandardCharsets.US_ASCII))
             
+            boolean feed = settings.fileFeed && settings.advertiseFeed
+            os.write("Feed: ${feed}\r\n".getBytes(StandardCharsets.US_ASCII))
+            
             os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
 
             DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
@@ -524,5 +536,58 @@ class ConnectionAcceptor {
             throw new Exception("Invalid IRC connection")
         chatServer.handle(e)
     }
+    
+    private void processFEED(Endpoint e) {
+        try {
+            byte[] EED = new byte[5];
+            DataInputStream dis = new DataInputStream(e.getInputStream())
+            dis.readFully(EED);
+            if (EED != "EED\r\n".getBytes(StandardCharsets.US_ASCII))
+                throw new Exception("Invalid FEED connection")
+
+            OutputStream os = e.getOutputStream()
+
+            Map<String, String> headers = DataUtil.readAllHeaders(dis)
+            if (!headers.containsKey("Persona"))
+                throw new Exception("Persona header missing")
+            Persona requestor = new Persona(new ByteArrayInputStream(Base64.decode(headers['Persona'])))
+            if (requestor.destination != e.destination)
+                throw new Exception("Requestor persona mismatch")
+
+            if (!settings.fileFeed) {
+                os.write("403 Not Allowed\r\n\r\n".getBytes(StandardCharsets.US_ASCII))
+                os.flush()
+                e.close()
+                return
+            }
+
+            long timestamp = 0
+            if (headers.containsKey("Timestamp")) {
+                timestamp = Long.parseLong(headers['Timestamp'])
+            }
+
+            List<SharedFile> published = fileManager.getPublishedSince(timestamp)
+
+            os.write("200 OK\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Count: ${published.size()}\r\n".getBytes(StandardCharsets.US_ASCII));
+            os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
+
+            DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
+            JsonOutput jsonOutput = new JsonOutput()
+            final long now = System.currentTimeMillis();
+            published.each {
+                it.hit(requestor, now, "Feed Update");
+                int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
+                def obj = FeedItems.sharedFileToObj(it, certificates)
+                def json = jsonOutput.toJson(obj)
+                dos.writeShort((short)json.length())
+                dos.write(json.getBytes(StandardCharsets.US_ASCII))
+            }
+            dos.flush()
+            dos.close()
+        } finally {
+            e.close()
+        }
+    }
 
 }

core/src/main/groovy/com/muwire/core/download/DownloadManager.groovy

@@ -1,6 +1,7 @@
 package com.muwire.core.download
 
 import com.muwire.core.connection.I2PConnector
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
 import com.muwire.core.files.FileDownloadedEvent
 import com.muwire.core.files.FileHasher
 import com.muwire.core.mesh.Mesh
@@ -62,11 +63,6 @@ public class DownloadManager {
 
 
     public void onUIDownloadEvent(UIDownloadEvent e) {
-
-        File incompletes = muSettings.incompleteLocation
-        if (incompletes == null)
-            incompletes = new File(home, "incompletes")
-        incompletes.mkdirs()
         
         def size = e.result[0].size
         def infohash = e.result[0].infohash
@@ -79,12 +75,29 @@ public class DownloadManager {
         destinations.addAll(e.sources)
         destinations.remove(me.destination)
 
-        Pieces pieces = getPieces(infohash, size, pieceSize, e.sequential)
+        doDownload(infohash, e.target, size, pieceSize, e.sequential, destinations)
 
-        def downloader = new Downloader(eventBus, this, me, e.target, size,
-            infohash, pieceSize, connector, destinations,
-            incompletes, pieces)
-        downloaders.put(infohash, downloader)
+    }
+    
+    public void onUIDownloadFeedItemEvent(UIDownloadFeedItemEvent e) {
+        Set<Destination> singleSource = new HashSet<>()
+        singleSource.add(e.item.getPublisher().getDestination())
+        doDownload(e.item.getInfoHash(), e.target, e.item.getSize(), e.item.getPieceSize(), 
+            e.sequential, singleSource)
+    }
+    
+    private void doDownload(InfoHash infoHash, File target, long size, int pieceSize, 
+        boolean sequential, Set<Destination> destinations) {
+        File incompletes = muSettings.incompleteLocation
+        if (incompletes == null)
+            incompletes = new File(home, "incompletes")
+        incompletes.mkdirs()
+        
+        Pieces pieces = getPieces(infoHash, size, pieceSize, sequential)
+        def downloader = new Downloader(eventBus, this, me, target, size,
+                infoHash, pieceSize, connector, destinations,
+                incompletes, pieces)
+        downloaders.put(infoHash, downloader)
         persistDownloaders()
         executor.execute({downloader.download()} as Runnable)
         eventBus.publish(new DownloadStartedEvent(downloader : downloader))
@@ -229,4 +242,8 @@ public class DownloadManager {
         downloaders.values().each { it.stop() }
         Downloader.executorService.shutdownNow()
     }
+    
+    public boolean isDownloading(InfoHash infoHash) {
+        downloaders.containsKey(infoHash)
+    }
 }

core/src/main/groovy/com/muwire/core/download/DownloadSession.groovy

@@ -183,15 +183,14 @@ class DownloadSession {
                 mapped.position(position)
 
                 byte[] tmp = new byte[0x1 << 13]
+                DataInputStream dis = new DataInputStream(is)
                 while(mapped.hasRemaining()) {
                     if (mapped.remaining() < tmp.length)
                         tmp = new byte[mapped.remaining()]
-                    int read = is.read(tmp)
-                    if (read == -1)
-                        throw new IOException()
+                    dis.readFully(tmp)
                     synchronized(this) {
-                        mapped.put(tmp, 0, read)
-                        dataSinceLastRead.addAndGet(read)
+                        mapped.put(tmp)
+                        dataSinceLastRead.addAndGet(tmp.length)
                         pieces.markPartial(piece, mapped.position())
                     }
                 }

core/src/main/groovy/com/muwire/core/download/Downloader.groovy

@@ -160,7 +160,7 @@ public class Downloader {
             long dataRead = dataSinceLastRead.getAndSet(0)
             long now = System.currentTimeMillis()
             if (now > lastSpeedRead)
-                currSpeed = (int) (dataRead * 1000.0 / (now - lastSpeedRead))
+                currSpeed = (int) (dataRead * 1000.0d / (now - lastSpeedRead))
             lastSpeedRead = now
         }
         

core/src/main/groovy/com/muwire/core/download/Pieces.groovy

@@ -2,10 +2,11 @@ package com.muwire.core.download
 
 class Pieces {
     private final BitSet done, claimed
-    private final int nPieces
+    final int nPieces
     private final float ratio
     private final Random random = new Random()
     private final Map<Integer,Integer> partials = new HashMap<>()
+    private int cachedDone;
 
     Pieces(int nPieces) {
         this(nPieces, 1.0f)
@@ -78,6 +79,7 @@ class Pieces {
         if (piece >= nPieces)
             throw new IllegalArgumentException("invalid piece marked as downloaded? $piece/$nPieces")
         done.set(piece)
+        cachedDone = done.cardinality();
         claimed.set(piece)
         partials.remove(piece)
     }
@@ -91,11 +93,11 @@ class Pieces {
     }
 
     synchronized boolean isComplete() {
-        done.cardinality() == nPieces
+        cachedDone == nPieces
     }
 
     synchronized int donePieces() {
-        done.cardinality()
+        cachedDone
     }
 
     synchronized boolean isDownloaded(int piece) {
@@ -104,6 +106,7 @@ class Pieces {
 
     synchronized void clearAll() {
         done.clear()
+        cachedDone = 0
         claimed.clear()
         partials.clear()
     }

core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy

@@ -0,0 +1,110 @@
+package com.muwire.core.filefeeds
+
+import java.util.logging.Level
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.Executor
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.zip.GZIPInputStream
+
+import com.muwire.core.EventBus
+import com.muwire.core.Persona
+import com.muwire.core.connection.Endpoint
+import com.muwire.core.connection.I2PConnector
+import com.muwire.core.util.DataUtil
+
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+@Log
+class FeedClient {
+    
+    private final I2PConnector connector
+    private final EventBus eventBus
+    private final Persona me
+    private final FeedManager feedManager
+    
+    private final ExecutorService feedFetcher = Executors.newCachedThreadPool() 
+    private final Timer feedUpdater = new Timer("feed-updater", true)
+    
+    FeedClient(I2PConnector connector, EventBus eventBus, Persona me, FeedManager feedManager) {
+        this.connector = connector
+        this.eventBus = eventBus
+        this.me = me
+        this.feedManager = feedManager
+    }
+    
+    private void start() {
+        feedUpdater.schedule({updateAnyFeeds()} as TimerTask, 60000, 60000)
+    }
+    
+    private void stop() {
+        feedUpdater.cancel()
+        feedFetcher.shutdown()
+    }
+    
+    private void updateAnyFeeds() {
+        feedManager.getFeedsToUpdate().each { feed ->
+            feedFetcher.execute({updateFeed(feed)} as Runnable)
+        }
+    }
+    
+    void onUIFeedUpdateEvent(UIFeedUpdateEvent e) {
+        Feed feed = feedManager.getFeed(e.host)
+        if (feed == null) {
+            log.severe("UI request to update non-existent feed " + e.host.getHumanReadableName())
+            return
+        }
+        
+        feedFetcher.execute({updateFeed(feed)} as Runnable)
+    }
+    
+    private void updateFeed(Feed feed) {
+        log.info("updating feed " + feed.getPublisher().getHumanReadableName())
+        Endpoint endpoint = null
+        try {
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.CONNECTING))
+            feed.setLastUpdateAttempt(System.currentTimeMillis())
+            endpoint = connector.connect(feed.getPublisher().getDestination())
+            OutputStream os = endpoint.getOutputStream()
+            os.write("FEED\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Persona:${me.toBase64()}\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Timestamp:${feed.getLastUpdated()}\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.flush()
+            
+            InputStream is = endpoint.getInputStream()
+            String code = DataUtil.readTillRN(is)
+            if (!code.startsWith("200"))
+                throw new IOException("Invalid code $code")
+                
+            // parse all headers
+            Map<String,String> headers = DataUtil.readAllHeaders(is)
+            
+            if (!headers.containsKey("Count"))
+                throw new IOException("No count header")
+            
+            int items = Integer.parseInt(headers['Count'])
+            
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FETCHING, totalItems: items))
+            
+            JsonSlurper slurper = new JsonSlurper()
+            DataInputStream dis = new DataInputStream(new GZIPInputStream(is))
+            for (int i = 0; i < items; i++) {
+                int size = dis.readUnsignedShort()
+                byte [] tmp = new byte[size]
+                dis.readFully(tmp)
+                def json = slurper.parse(tmp)
+                FeedItem item = FeedItems.objToFeedItem(json, feed.getPublisher())
+                eventBus.publish(new FeedItemFetchedEvent(item: item))
+            }
+            
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FINISHED))
+        } catch (Exception bad) {
+            log.log(Level.WARNING, "Feed update failed", bad)
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FAILED)) 
+        } finally {
+            endpoint?.close()
+        }
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedFetchEvent.groovy

@@ -0,0 +1,10 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class FeedFetchEvent extends Event {
+    Persona host
+    FeedFetchStatus status
+    int totalItems
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItemFetchedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedItemFetchedEvent extends Event {
+    FeedItem item
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItemLoadedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedItemLoadedEvent extends Event {
+    FeedItem item
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItems.groovy

@@ -0,0 +1,79 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.InfoHash
+import com.muwire.core.Persona
+import com.muwire.core.SharedFile
+import com.muwire.core.files.FileHasher
+import com.muwire.core.util.DataUtil
+
+import net.i2p.data.Base64
+
+class FeedItems {
+    
+    public static def sharedFileToObj(SharedFile sf, int certificates) {
+        def json = [:]
+        json.type = "FeedItem"
+        json.version = 1
+        json.name = Base64.encode(DataUtil.encodei18nString(sf.getFile().getName()))
+        json.infoHash = Base64.encode(sf.getRoot())
+        json.size = sf.getCachedLength()
+        json.pieceSize = sf.getPieceSize()
+        
+        if (sf.getComment() != null)
+            json.comment = sf.getComment()
+        
+        json.certificates = certificates
+        
+        json.timestamp = sf.getPublishedTimestamp()
+        
+        json
+    } 
+    
+    public static FeedItem objToFeedItem(def obj, Persona publisher) throws InvalidFeedItemException {
+        if (obj.timestamp == null)
+            throw new InvalidFeedItemException("No timestamp");
+        if (obj.name == null)
+            throw new InvalidFeedItemException("No name");
+        if (obj.size == null || obj.size <= 0 || obj.size > FileHasher.MAX_SIZE)
+            throw new InvalidFeedItemException("length missing or invalid ${obj.size}")
+        if (obj.pieceSize == null || obj.pieceSize < FileHasher.MIN_PIECE_SIZE_POW2 || obj.pieceSize > FileHasher.MAX_PIECE_SIZE_POW2)
+            throw new InvalidFeedItemException("piece size missing or invalid ${obj.pieceSize}")
+        if (obj.infoHash == null)
+            throw new InvalidFeedItemException("Infohash missing")
+        
+            
+        InfoHash infoHash
+        try {
+            infoHash = new InfoHash(Base64.decode(obj.infoHash))
+        } catch (Exception bad) {
+            throw new InvalidFeedItemException("Invalid infohash", bad)
+        }
+        
+        String name
+        try {
+            name = DataUtil.readi18nString(Base64.decode(obj.name))
+        } catch (Exception bad) {
+            throw new InvalidFeedItemException("Invalid name", bad)
+        }
+        
+        int certificates = 0
+        if (obj.certificates != null)
+            certificates = obj.certificates
+        
+        new FeedItem(publisher, obj.timestamp, name, obj.size, obj.pieceSize, infoHash, certificates, obj.comment)
+    }
+    
+    public static def feedItemToObj(FeedItem item) {
+        def json = [:]
+        json.type = "FeedItem"
+        json.version = 1
+        json.name = Base64.encode(DataUtil.encodei18nString(item.getName()))
+        json.infoHash = Base64.encode(item.getInfoHash().getRoot())
+        json.size = item.getSize()
+        json.pieceSize = item.getPieceSize()
+        json.timestamp = item.getTimestamp()
+        json.certificates = item.getCertificates()
+        json.comment = item.getComment()
+        json
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedLoadedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedLoadedEvent extends Event {
+    Feed feed
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy

@@ -0,0 +1,225 @@
+package com.muwire.core.filefeeds
+
+import java.nio.file.Files
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadFactory
+import java.util.stream.Collectors
+
+import com.muwire.core.EventBus
+import com.muwire.core.Persona
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+import net.i2p.data.Base64
+import net.i2p.util.ConcurrentHashSet
+
+@Log
+class FeedManager {
+    
+    private final EventBus eventBus
+    private final File metadataFolder, itemsFolder
+    private final Map<Persona, Feed> feeds = new ConcurrentHashMap<>()
+    private final Map<Persona, Set<FeedItem>> feedItems = new ConcurrentHashMap<>()
+    
+    private final ExecutorService persister = Executors.newSingleThreadExecutor({r ->
+        new Thread(r, "feed persister")
+    } as ThreadFactory)
+    
+    
+    FeedManager(EventBus eventBus, File home) {
+        this.eventBus = eventBus
+        File feedsFolder = new File(home, "filefeeds")
+        if (!feedsFolder.exists())
+            feedsFolder.mkdir()
+        this.metadataFolder = new File(feedsFolder, "metadata")
+        if (!metadataFolder.exists())
+            metadataFolder.mkdir()
+        this.itemsFolder = new File(feedsFolder, "items")
+        if (!itemsFolder.exists())
+            itemsFolder.mkdir()
+    }
+    
+    public Feed getFeed(Persona persona) {
+        feeds.get(persona)
+    }
+    
+    public Set<FeedItem> getFeedItems(Persona persona) {
+        feedItems.getOrDefault(persona, Collections.emptySet())
+    }
+    
+    public List<Feed> getFeedsToUpdate() {
+        long now = System.currentTimeMillis()
+        feeds.values().stream().
+            filter({Feed f -> !f.getStatus().isActive()}).
+            filter({Feed f -> f.getLastUpdateAttempt() + f.getUpdateInterval() <= now})
+            .collect(Collectors.toList())
+    }
+    
+    void start() {
+        log.info("starting feed manager")
+        persister.submit({loadFeeds()} as Runnable)
+        persister.submit({loadItems()} as Runnable)
+    }
+    
+    void stop() {
+        persister.shutdown()
+    }
+    
+    private void loadFeeds() {
+        def slurper = new JsonSlurper()
+        Files.walk(metadataFolder.toPath()).
+            filter( { it.getFileName().toString().endsWith(".json")}).
+            forEach( {
+                def parsed = slurper.parse(it.toFile())
+                Persona publisher = new Persona(new ByteArrayInputStream(Base64.decode(parsed.publisher)))
+                Feed feed = new Feed(publisher)
+                feed.setUpdateInterval(parsed.updateInterval)
+                feed.setLastUpdated(parsed.lastUpdated)
+                feed.setLastUpdateAttempt(parsed.lastUpdateAttempt)
+                feed.setItemsToKeep(parsed.itemsToKeep)
+                feed.setAutoDownload(parsed.autoDownload)
+                feed.setSequential(parsed.sequential)
+                
+                feed.setStatus(FeedFetchStatus.IDLE)
+                
+                feeds.put(feed.getPublisher(), feed)
+                
+                eventBus.publish(new FeedLoadedEvent(feed : feed))
+            })     
+    }
+    
+    private void loadItems() {
+        def slurper = new JsonSlurper()
+        feeds.keySet().each { persona ->
+            File itemsFile = getItemsFile(feeds[persona])
+            if (!itemsFile.exists())
+                return // no items yet?
+            itemsFile.eachLine { line ->
+                def parsed = slurper.parseText(line)
+                FeedItem item = FeedItems.objToFeedItem(parsed, persona)
+                Set<FeedItem> items = feedItems.get(persona)
+                if (items == null) {
+                    items = new ConcurrentHashSet<>()
+                    feedItems.put(persona, items)
+                }
+                items.add(item)
+                eventBus.publish(new FeedItemLoadedEvent(item : item))
+            }
+        }
+    }
+    
+    void onFeedItemFetchedEvent(FeedItemFetchedEvent e) {
+        Set<FeedItem> set = feedItems.get(e.item.getPublisher())
+        if (set == null) {
+            set = new ConcurrentHashSet<>()
+            feedItems.put(e.getItem().getPublisher(), set)
+        }
+        set.add(e.item)
+    }
+    
+    void onFeedFetchEvent(FeedFetchEvent e) {
+        
+        Feed feed = feeds.get(e.host)
+        if (feed == null) {
+            log.severe("Fetching non-existent feed " + e.host.getHumanReadableName())
+            return
+        }
+        
+        feed.setStatus(e.status)
+        
+        if (e.status.isActive())
+            return
+        
+        if (e.status == FeedFetchStatus.FINISHED) {
+            feed.setStatus(FeedFetchStatus.IDLE)
+            feed.setLastUpdated(e.getTimestamp())
+        }
+        // save feed items, then save feed.  This will save partial fetches too
+        // which is ok because the items are stored in a Set
+        persister.submit({saveFeedItems(e.host)} as Runnable)
+        persister.submit({saveFeedMetadata(feed)} as Runnable)
+    }
+    
+    void onUIFeedConfigurationEvent(UIFeedConfigurationEvent e) {
+        feeds.put(e.feed.getPublisher(), e.feed)
+        persister.submit({saveFeedMetadata(e.feed)} as Runnable)
+    }
+    
+    void onUIFeedDeletedEvent(UIFeedDeletedEvent e) {
+        Feed f = feeds.get(e.host)
+        if (f == null) {
+            log.severe("Deleting a non-existing feed " + e.host.getHumanReadableName())
+            return
+        }
+        persister.submit({deleteFeed(f)} as Runnable)
+    }
+    
+    private void saveFeedItems(Persona publisher) {
+        Set<FeedItem> set = feedItems.get(publisher)
+        if (set == null)
+            return // can happen if nothing was published
+        
+        Feed feed = feeds[publisher]
+        if (feed == null) {
+            log.severe("Persisting items for non-existing feed " + publisher.getHumanReadableName())
+            return
+        }   
+              
+        if (feed.getItemsToKeep() == 0)
+            return
+             
+        List<FeedItem> list = new ArrayList<>(set)
+        if (feed.getItemsToKeep() > 0 && list.size() > feed.getItemsToKeep()) {
+            log.info("will persist ${feed.getItemsToKeep()}/${list.size()} items")
+            list.sort({l, r ->
+                Long.compare(r.getTimestamp(), l.getTimestamp())
+            } as Comparator<FeedItem>)
+            list = list[0..feed.getItemsToKeep() - 1]
+        }
+        
+        
+        File itemsFile = getItemsFile(feed)
+        itemsFile.withPrintWriter { writer -> 
+            list.each { item -> 
+                def obj = FeedItems.feedItemToObj(item)
+                def json = JsonOutput.toJson(obj)
+                writer.println(json)
+            }
+        }
+    }
+    
+    private void saveFeedMetadata(Feed feed) {
+        File metadataFile = getMetadataFile(feed)
+        metadataFile.withPrintWriter { writer ->
+            def json = [:]
+            json.publisher = feed.getPublisher().toBase64()
+            json.itemsToKeep = feed.getItemsToKeep()
+            json.lastUpdated = feed.getLastUpdated()
+            json.updateInterval = feed.getUpdateInterval()
+            json.autoDownload = feed.isAutoDownload()
+            json.sequential = feed.isSequential()
+            json.lastUpdateAttempt = feed.getLastUpdateAttempt()
+            json = JsonOutput.toJson(json)
+            writer.println(json)
+        }
+    }
+    
+    private void deleteFeed(Feed feed) {
+        feeds.remove(feed.getPublisher())
+        feedItems.remove(feed.getPublisher())
+        getItemsFile(feed).delete()
+        getMetadataFile(feed).delete()
+    }
+    
+    private File getItemsFile(Feed feed) {
+        return new File(itemsFolder, feed.getPublisher().destination.toBase32() + ".json")
+    }
+    
+    private File getMetadataFile(Feed feed) {
+        return new File(metadataFolder, feed.getPublisher().destination.toBase32() + ".json")
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIDownloadFeedItemEvent.groovy

@@ -0,0 +1,9 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class UIDownloadFeedItemEvent extends Event {
+    FeedItem item
+    File target
+    boolean sequential
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedConfigurationEvent.groovy

@@ -0,0 +1,12 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+/**
+ * Emitted when configuration of a feed changes.  
+ * The object should already contain the updated values.
+ */
+class UIFeedConfigurationEvent extends Event {
+    Feed feed
+    boolean newFeed
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedDeletedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class UIFeedDeletedEvent extends Event {
+    Persona host
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedUpdateEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class UIFeedUpdateEvent extends Event {
+    Persona host
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFilePublishedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.SharedFile
+
+class UIFilePublishedEvent extends Event {
+    SharedFile sf
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFileUnpublishedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.SharedFile
+
+class UIFileUnpublishedEvent extends Event {
+    SharedFile sf
+}

core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy

@@ -47,7 +47,7 @@ abstract class BasePersisterService extends Service{
         int pieceSize = 0
         if (json.pieceSize != null)
             pieceSize = json.pieceSize
-
+            
         if (json.sources != null) {
             List sources = (List)json.sources
             Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
@@ -94,10 +94,19 @@ abstract class BasePersisterService extends Service{
         if (json.pieceSize != null)
             pieceSize = json.pieceSize
 
+        boolean published = false
+        long publishedTimestamp = -1
+        if (json.published != null && json.published) {
+            published = true
+            publishedTimestamp = json.publishedTimestamp
+        }
+
         if (json.sources != null) {
             List sources = (List)json.sources
             Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
             DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet)
+            if (published)
+                df.publish(publishedTimestamp)
             df.setComment(json.comment)
             return new FileLoadedEvent(loadedFile : df, infoHash: ih)
         }
@@ -105,6 +114,8 @@ abstract class BasePersisterService extends Service{
 
         SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize)
         sf.setComment(json.comment)
+        if (published)
+            sf.publish(publishedTimestamp)
         if (json.downloaders != null)
             sf.getDownloaders().addAll(json.downloaders)
         if (json.searchers != null) {
@@ -146,6 +157,11 @@ abstract class BasePersisterService extends Service{
         if (sf instanceof DownloadedFile) {
             json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList())
         }
+        
+        if (sf.isPublished()) {
+            json.published = true
+            json.publishedTimestamp = sf.getPublishedTimestamp()
+        }
 
         json
     }

core/src/main/groovy/com/muwire/core/files/DirectoryUnsharedEvent.groovy

@@ -4,8 +4,9 @@ import com.muwire.core.Event
 
 class DirectoryUnsharedEvent extends Event {
     File directory
+    boolean deleted
     
     public String toString() {
-        super.toString() + " unshared directory "+ directory.toString()
+        super.toString() + " unshared directory "+ directory.toString() + " deleted $deleted"
     }
 }

core/src/main/groovy/com/muwire/core/files/DirectoryWatcher.groovy

@@ -15,6 +15,9 @@ import java.util.concurrent.ConcurrentHashMap
 import com.muwire.core.EventBus
 import com.muwire.core.MuWireSettings
 import com.muwire.core.SharedFile
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectoryConvertedEvent
+import com.muwire.core.files.directories.WatchedDirectoryManager
 
 import groovy.util.logging.Log
 import net.i2p.util.SystemVersion
@@ -33,27 +36,27 @@ class DirectoryWatcher {
     }
 
     private final File home
-    private final MuWireSettings muOptions
     private final EventBus eventBus
     private final FileManager fileManager
+    private final WatchedDirectoryManager watchedDirectoryManager
     private final Thread watcherThread, publisherThread
     private final Map<File, Long> waitingFiles = new ConcurrentHashMap<>()
     private final Map<File, WatchKey> watchedDirectories = new ConcurrentHashMap<>()
     private WatchService watchService
     private volatile boolean shutdown
 
-    DirectoryWatcher(EventBus eventBus, FileManager fileManager, File home, MuWireSettings muOptions) {
+    DirectoryWatcher(EventBus eventBus, FileManager fileManager, File home, WatchedDirectoryManager watchedDirectoryManager) {
         this.home = home
-        this.muOptions = muOptions
         this.eventBus = eventBus
         this.fileManager = fileManager
+        this.watchedDirectoryManager = watchedDirectoryManager
         this.watcherThread = new Thread({watch() } as Runnable, "directory-watcher")
         watcherThread.setDaemon(true)
         this.publisherThread = new Thread({publish()} as Runnable, "watched-files-publisher")
         publisherThread.setDaemon(true)
     }
 
-    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
+    void onWatchedDirectoryConvertedEvent(WatchedDirectoryConvertedEvent e) {
         watchService = FileSystems.getDefault().newWatchService()
         watcherThread.start()
         publisherThread.start()
@@ -71,26 +74,26 @@ class DirectoryWatcher {
         Path path = canonical.toPath()
         WatchKey wk = path.register(watchService, kinds)
         watchedDirectories.put(canonical, wk)
-        
-        if (muOptions.watchedDirectories.add(canonical.toString()))
-            saveMuSettings()
     }
 
     void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
         WatchKey wk = watchedDirectories.remove(e.directory)
         wk?.cancel()
-        
-        if (muOptions.watchedDirectories.remove(e.directory.toString()))
-            saveMuSettings()
     }
     
-    private void saveMuSettings() {
-        File muSettingsFile = new File(home, "MuWire.properties")
-        muSettingsFile.withPrintWriter("UTF-8", {
-            muOptions.write(it)
-        })
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        if (watchService == null)
+            return // still converting
+        if (!e.autoWatch) {
+            WatchKey wk = watchedDirectories.remove(e.directory)
+            wk?.cancel()
+        } else if (!watchedDirectories.containsKey(e.directory)) {
+            Path path = e.directory.toPath()
+            def wk = path.register(watchService, kinds)
+            watchedDirectories.put(e.directory, wk)
+        } // else it was already watched
     }
-
+    
     private void watch() {
         try {
             while(!shutdown) {
@@ -115,7 +118,7 @@ class DirectoryWatcher {
         File f= join(parent, path)
         log.fine("created entry $f")
         if (f.isDirectory())
-            f.toPath().register(watchService, kinds)
+            eventBus.publish(new FileSharedEvent(file : f, fromWatch : true))
         else
             waitingFiles.put(f, System.currentTimeMillis())
     }
@@ -133,6 +136,10 @@ class DirectoryWatcher {
         SharedFile sf = fileManager.fileToSharedFile.get(f)
         if (sf != null)
             eventBus.publish(new FileUnsharedEvent(unsharedFile : sf, deleted : true))
+        else if (watchedDirectoryManager.isWatched(f)) 
+            eventBus.publish(new DirectoryUnsharedEvent(directory : f, deleted : true))
+        else
+            log.fine("Entry was not relevant");
     }
 
     private static File join(Path parent, Path path) {
@@ -149,7 +156,7 @@ class DirectoryWatcher {
                 waitingFiles.each { file, timestamp ->
                     if (now - timestamp > WAIT_TIME) {
                         log.fine("publishing file $file")
-                        eventBus.publish new FileSharedEvent(file : file)
+                        eventBus.publish new FileSharedEvent(file : file, fromWatch: true)
                         published << file
                     }
                 }

core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy

@@ -12,7 +12,7 @@ class FileHashedEvent extends Event {
 
     @Override
     public String toString() {
-        super.toString() + " sharedFile " + sharedFile?.file.getAbsolutePath() + " error: $error"
+        super.toString() + " sharedFile " + sharedFile?.file?.getAbsolutePath() + " error: $error"
     }
 
 }

core/src/main/groovy/com/muwire/core/files/FileManager.groovy

@@ -1,5 +1,8 @@
 package com.muwire.core.files
 
+import java.util.stream.Collectors
+import java.util.stream.Stream
+
 import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
@@ -25,6 +28,7 @@ class FileManager {
     final Map<String, Set<File>> commentToFile = new HashMap<>()
     final SearchIndex index = new SearchIndex()
     final FileTree<Void> negativeTree = new FileTree<>()
+    final FileTree<SharedFile> positiveTree = new FileTree<>()
     final Set<File> sideCarFiles = new HashSet<>()
 
     FileManager(EventBus eventBus, MuWireSettings settings) {
@@ -84,6 +88,7 @@ class FileManager {
         }
         existing.add(sf)
         fileToSharedFile.put(sf.file, sf)
+        positiveTree.add(sf.file, sf);
         
         negativeTree.remove(sf.file)
         String parent = sf.getFile().getParent()
@@ -127,6 +132,7 @@ class FileManager {
         }
 
         fileToSharedFile.remove(sf.file)
+        positiveTree.remove(sf.file)
         if (!e.deleted && negativeTree.fileToNode.containsKey(sf.file.getParentFile())) {
             negativeTree.add(sf.file,null)
             saveNegativeTree()
@@ -190,6 +196,10 @@ class FileManager {
     Set<SharedFile> getSharedFiles(byte []root) {
             return rootToFiles.get(new InfoHash(root))
     }
+    
+    boolean isShared(InfoHash infoHash) {
+        rootToFiles.containsKey(infoHash)
+    }
 
     void onSearchEvent(SearchEvent e) {
         // hash takes precedence
@@ -239,14 +249,26 @@ class FileManager {
     void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
         negativeTree.remove(e.directory)
         saveNegativeTree()
-        e.directory.listFiles().each {
-            if (it.isDirectory())
-                eventBus.publish(new DirectoryUnsharedEvent(directory : it))
-            else {
-                SharedFile sf = fileToSharedFile.get(it)
-                if (sf != null)
-                    eventBus.publish(new FileUnsharedEvent(unsharedFile : sf))
+        if (!e.deleted) {
+            e.directory.listFiles().each {
+                if (it.isDirectory())
+                    eventBus.publish(new DirectoryUnsharedEvent(directory : it))
+                else {
+                    SharedFile sf = fileToSharedFile.get(it)
+                    if (sf != null)
+                        eventBus.publish(new FileUnsharedEvent(unsharedFile : sf))
+                }
             }
+        } else {
+             def cb = new DirDeletionCallback()
+             positiveTree.traverse(e.directory, cb)
+             positiveTree.remove(e.directory)
+             cb.unsharedFiles.each { 
+                 eventBus.publish(new FileUnsharedEvent(unsharedFile : it, deleted: true))
+             }
+             cb.subDirs.each {
+                 eventBus.publish(new DirectoryUnsharedEvent(directory : it, deleted : true))
+             }
         }
     }
     
@@ -254,4 +276,34 @@ class FileManager {
         settings.negativeFileTree.clear()
         settings.negativeFileTree.addAll(negativeTree.fileToNode.keySet().collect { it.getAbsolutePath() })
     }
+    
+    public List<SharedFile> getPublishedSince(long timestamp) {
+        synchronized(fileToSharedFile) {
+            fileToSharedFile.values().stream().
+                    filter({sf -> sf.isPublished()}).
+                    filter({sf -> sf.getPublishedTimestamp() >= timestamp}).
+                    collect(Collectors.toList())
+        }
+    }
+    
+    private static class DirDeletionCallback implements FileTreeCallback<SharedFile> {
+        
+        final List<File> subDirs = new ArrayList<>()
+        final List<SharedFile> unsharedFiles = new ArrayList<>()
+
+        @Override
+        public void onDirectoryEnter(File file) {
+            subDirs.add(file)
+        }
+
+        @Override
+        public void onDirectoryLeave() {
+        }
+
+        @Override
+        public void onFile(File file, SharedFile value) {
+            unsharedFiles << value
+        }
+        
+    }
 }

core/src/main/groovy/com/muwire/core/files/FileSharedEvent.groovy

@@ -5,9 +5,10 @@ import com.muwire.core.Event
 class FileSharedEvent extends Event {
 
     File file
+    boolean fromWatch
 
     @Override
     public String toString() {
-        return super.toString() + " file: "+file.getAbsolutePath()
+        return super.toString() + " file: "+file.getAbsolutePath() + " fromWatch: $fromWatch"
     }
 }

core/src/main/groovy/com/muwire/core/files/FileTree.groovy

@@ -23,6 +23,7 @@ class FileTree<T> {
             if (existing == null) {
                 existing = new TreeNode()
                 existing.file = element
+                existing.isFile = element.isFile()
                 existing.parent = current
                 fileToNode.put(element, existing)
                 current.children.add(existing)
@@ -64,7 +65,7 @@ class FileTree<T> {
     private void doTraverse(TreeNode<T> node, FileTreeCallback<T> callback) {
         boolean leave = false
         if (node.file != null) {
-            if (node.file.isFile())
+            if (node.isFile)
                 callback.onFile(node.file, node.value)
             else {
                 leave = true
@@ -88,7 +89,7 @@ class FileTree<T> {
             node = fileToNode.get(parent)
             
         node.children.each { 
-            if (it.file.isFile())
+            if (it.isFile)
                 callback.onFile(it.file, it.value)
             else
                 callback.onDirectory(it.file)
@@ -98,6 +99,7 @@ class FileTree<T> {
     public static class TreeNode<T> {
         TreeNode parent
         File file
+        boolean isFile
         T value;
         final Set<TreeNode> children = new HashSet<>()
         

core/src/main/groovy/com/muwire/core/files/HasherService.groovy

@@ -53,7 +53,6 @@ class HasherService {
 
     private void process(File f) {
         if (f.isDirectory()) {
-            eventBus.publish(new DirectoryWatchedEvent(directory : f))
             f.listFiles().each {
                 eventBus.publish new FileSharedEvent(file: it)
             }

core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy

@@ -1,6 +1,9 @@
 package com.muwire.core.files
 
 import com.muwire.core.*
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
+
 import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import groovy.util.logging.Log
@@ -56,11 +59,15 @@ class PersisterFolderService extends BasePersisterService {
     }
 
     void onFileHashedEvent(FileHashedEvent hashedEvent) {
+        if (core.getMuOptions().getAutoPublishSharedFiles() && hashedEvent.sharedFile != null) 
+            hashedEvent.sharedFile.publish(System.currentTimeMillis())
         persistFile(hashedEvent.sharedFile, hashedEvent.infoHash)
     }
 
     void onFileDownloadedEvent(FileDownloadedEvent downloadedEvent) {
         if (core.getMuOptions().getShareDownloadedFiles()) {
+            if (core.getMuOptions().getAutoPublishSharedFiles())
+                downloadedEvent.downloadedFile.publish(System.currentTimeMillis())
             persistFile(downloadedEvent.downloadedFile, downloadedEvent.infoHash)
         }
     }
@@ -92,6 +99,14 @@ class PersisterFolderService extends BasePersisterService {
     void onUICommentEvent(UICommentEvent e) {
         persistFile(e.sharedFile,null)
     }
+    
+    void onUIFilePublishedEvent(UIFilePublishedEvent e) {
+        persistFile(e.sf, null)
+    }
+    
+    void onUIFileUnpublishedEvent(UIFileUnpublishedEvent e) {
+        persistFile(e.sf, null)
+    }
 
     void load() {
         log.fine("Loading...")
@@ -101,7 +116,7 @@ class PersisterFolderService extends BasePersisterService {
             try {
                 _load()
             }
-            catch (IllegalArgumentException e) {
+            catch (Exception e) {
                 log.log(Level.WARNING, "couldn't load files", e)
             }
         } else {

core/src/main/groovy/com/muwire/core/files/PersisterService.groovy

@@ -62,7 +62,7 @@ class PersisterService extends BasePersisterService {
                         new File(location.absolutePath + ".bak")
                 )
                 listener.publish(new PersisterDoneEvent())
-            } catch (IllegalArgumentException e) {
+            } catch (Exception e) {
                 log.log(Level.WARNING, "couldn't load files",e)
             }
         } else {

core/src/main/groovy/com/muwire/core/files/directories/UISyncDirectoryEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class UISyncDirectoryEvent extends Event {
+    File directory
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectory.groovy

@@ -0,0 +1,37 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.util.DataUtil
+
+import net.i2p.data.Base64
+
+class WatchedDirectory {
+    final File directory
+    final String encodedName
+    boolean autoWatch
+    int syncInterval
+    long lastSync
+    
+    WatchedDirectory(File directory) {
+        this.directory = directory.getCanonicalFile()
+        this.encodedName = Base64.encode(DataUtil.encodei18nString(directory.getAbsolutePath()))
+    }
+    
+    def toJson() {
+        def rv = [:]
+        rv.directory = encodedName
+        rv.autoWatch = autoWatch
+        rv.syncInterval = syncInterval
+        rv.lastSync = lastSync
+        rv
+    }
+    
+    static WatchedDirectory fromJson(def json) {
+        String dirName = DataUtil.readi18nString(Base64.decode(json.directory))
+        File dir = new File(dirName)
+        def rv = new WatchedDirectory(dir)
+        rv.autoWatch = json.autoWatch
+        rv.syncInterval = json.syncInterval
+        rv.lastSync = json.lastSync
+        rv
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConfigurationEvent.groovy

@@ -0,0 +1,9 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class WatchedDirectoryConfigurationEvent extends Event {
+    File directory
+    boolean autoWatch
+    int syncInterval
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConvertedEvent.groovy

@@ -0,0 +1,10 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+/**
+ * Emitted when converting an old watched directory entry to the
+ * new format.
+ */
+class WatchedDirectoryConvertedEvent extends Event {
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConverter.groovy

@@ -0,0 +1,27 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Core
+import com.muwire.core.files.AllFilesLoadedEvent
+
+/**
+ * converts the setting-based format to new folder-based format.
+ */
+class WatchedDirectoryConverter {
+    
+    private final Core core
+    
+    WatchedDirectoryConverter(Core core) {
+        this.core = core
+    }
+    
+    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
+        core.getMuOptions().getWatchedDirectories().each {
+            File directory = new File(it)
+            directory = directory.getCanonicalFile() 
+            core.eventBus.publish(new WatchedDirectoryConfigurationEvent(directory : directory, autoWatch: true))
+        }
+        core.getMuOptions().getWatchedDirectories().clear()
+        core.saveMuSettings()
+        core.eventBus.publish(new WatchedDirectoryConvertedEvent())
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryManager.groovy

@@ -0,0 +1,220 @@
+package com.muwire.core.files.directories
+
+import java.nio.file.Files
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadFactory
+import java.util.stream.Stream
+
+import com.muwire.core.EventBus
+import com.muwire.core.SharedFile
+import com.muwire.core.files.DirectoryUnsharedEvent
+import com.muwire.core.files.DirectoryWatchedEvent
+import com.muwire.core.files.FileListCallback
+import com.muwire.core.files.FileManager
+import com.muwire.core.files.FileSharedEvent
+import com.muwire.core.files.FileUnsharedEvent
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+@Log
+class WatchedDirectoryManager {
+    
+    private final File home
+    private final EventBus eventBus
+    private final FileManager fileManager
+    
+    private final Map<File, WatchedDirectory> watchedDirs = new ConcurrentHashMap<>()
+    
+    private final ExecutorService diskIO = Executors.newSingleThreadExecutor({r -> 
+        Thread t = new Thread(r, "disk-io")
+        t.setDaemon(true)
+        t
+    } as ThreadFactory)
+    
+    private final Timer timer = new Timer("directory-timer", true)
+    
+    private boolean converting = true
+    
+    WatchedDirectoryManager(File home, EventBus eventBus, FileManager fileManager) {
+        this.home = new File(home, "directories")
+        this.home.mkdir()
+        this.eventBus = eventBus
+        this.fileManager = fileManager
+    }
+    
+    public boolean isWatched(File f) {
+        watchedDirs.containsKey(f)
+    }
+    
+    public Stream<WatchedDirectory> getWatchedDirsStream() {
+        watchedDirs.values().stream()
+    }
+    
+    public void shutdown() {
+        diskIO.shutdown()
+        timer.cancel()
+    }
+    
+    void onUISyncDirectoryEvent(UISyncDirectoryEvent e) {
+        def wd = watchedDirs.get(e.directory)
+        if (wd == null) {
+            log.warning("Got a sync event for non-watched dir ${e.directory}")
+            return
+        }
+        diskIO.submit({sync(wd, System.currentTimeMillis())} as Runnable)
+    }
+    
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        if (converting) {
+            def newDir = new WatchedDirectory(e.directory)
+            // conversion is always autowatch really
+            newDir.autoWatch = e.autoWatch
+            persist(newDir)
+        } else {
+            def wd = watchedDirs.get(e.directory)
+            if (wd == null) {
+                log.severe("got a configuration event for a non-watched directory ${e.directory}")
+                return
+            }
+            wd.autoWatch = e.autoWatch
+            wd.syncInterval = e.syncInterval
+            persist(wd)
+        }
+    }
+    
+    void onWatchedDirectoryConvertedEvent(WatchedDirectoryConvertedEvent e) {
+        converting = false
+        diskIO.submit({
+            def slurper = new JsonSlurper()
+            Files.walk(home.toPath()).filter({
+                it.getFileName().toString().endsWith(".json")
+            }).
+            forEach {
+                def parsed = slurper.parse(it.toFile())
+                WatchedDirectory wd = WatchedDirectory.fromJson(parsed)
+                watchedDirs.put(wd.directory, wd)
+            }
+            watchedDirs.values().stream().filter({it.autoWatch}).forEach {
+                eventBus.publish(new DirectoryWatchedEvent(directory : it.directory))
+                eventBus.publish(new FileSharedEvent(file : it.directory))
+            }
+            timer.schedule({sync()} as TimerTask, 1000, 1000)
+        } as Runnable)
+    }
+    
+    private void persist(WatchedDirectory dir) {
+        diskIO.submit({doPersist(dir)} as Runnable)
+    }
+    
+    private void doPersist(WatchedDirectory dir) {
+        def json = JsonOutput.toJson(dir.toJson())
+        def targetFile = new File(home, dir.getEncodedName() + ".json")
+        targetFile.text = json
+    }
+    
+    void onFileSharedEvent(FileSharedEvent e) {
+        if (e.file.isFile() || watchedDirs.containsKey(e.file))
+            return
+        
+        def wd = new WatchedDirectory(e.file)
+        if (e.fromWatch) {
+            // parent should be already watched, copy settings
+            def parent = watchedDirs.get(e.file.getParentFile())
+            if (parent == null) {
+                log.severe("watching found a directory without a watched parent? ${e.file}")
+                return
+            }
+            wd.autoWatch = parent.autoWatch
+            wd.syncInterval = parent.syncInterval
+        } else
+            wd.autoWatch = true
+        
+        watchedDirs.put(wd.directory, wd)
+        persist(wd)
+        if (wd.autoWatch)
+            eventBus.publish(new DirectoryWatchedEvent(directory: wd.directory))
+    }
+    
+    void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
+        def wd = watchedDirs.remove(e.directory)
+        if (wd == null) {
+            log.warning("unshared a directory that wasn't watched? ${e.directory}")
+            return
+        }
+        
+        File persistFile = new File(home, wd.getEncodedName() + ".json")
+        persistFile.delete()
+    }
+    
+    private void sync() {
+        long now = System.currentTimeMillis()
+        watchedDirs.values().stream().
+            filter({!it.autoWatch}).
+            filter({it.syncInterval > 0}).
+            filter({it.lastSync + it.syncInterval * 1000 < now}).
+            forEach({wd -> diskIO.submit({sync(wd, now)} as Runnable )})
+    }
+    
+    private void sync(WatchedDirectory wd, long now) {
+        log.fine("syncing ${wd.directory}")
+        wd.lastSync = now
+        doPersist(wd)
+        eventBus.publish(new WatchedDirectorySyncEvent(directory: wd.directory, when: now))
+        
+        def cb = new DirSyncCallback()
+        fileManager.positiveTree.list(wd.directory, cb)
+        
+        Set<File> filesOnFS = new HashSet<>()
+        Set<File> dirsOnFS = new HashSet<>()
+        wd.directory.listFiles().each {
+            File canonical = it.getCanonicalFile() 
+            if (canonical.isFile())
+                filesOnFS.add(canonical)
+            else
+                dirsOnFS.add(canonical)
+        }
+        
+        Set<File> addedFiles = new HashSet<>(filesOnFS)
+        addedFiles.removeAll(cb.files)
+        addedFiles.each { 
+            eventBus.publish(new FileSharedEvent(file : it, fromWatch : true))
+        }
+        Set<File> addedDirs = new HashSet<>(dirsOnFS)
+        addedDirs.removeAll(cb.dirs)
+        addedDirs.each { 
+            eventBus.publish(new FileSharedEvent(file : it, fromWatch : true))
+        }
+        
+        Set<File> deletedFiles = new HashSet<>(cb.files)
+        deletedFiles.removeAll(filesOnFS)
+        deletedFiles.each {
+            eventBus.publish(new FileUnsharedEvent(unsharedFile : fileManager.getFileToSharedFile().get(it), deleted : true))
+        }
+        Set<File> deletedDirs = new HashSet<>(cb.dirs)
+        deletedDirs.removeAll(dirsOnFS)
+        deletedDirs.each {
+            eventBus.publish(new DirectoryUnsharedEvent(directory : it, deleted: true))
+        }
+    }
+    
+    private static class DirSyncCallback implements FileListCallback<SharedFile> {
+        
+        private final Set<File> files = new HashSet<>()
+        private final Set<File> dirs = new HashSet<>()
+
+        @Override
+        public void onFile(File f, SharedFile value) {
+            files.add(f)
+        }
+
+        @Override
+        public void onDirectory(File f) {
+            dirs.add(f)
+        }
+        
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectorySyncEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class WatchedDirectorySyncEvent extends Event {
+    File directory
+    long when
+}

core/src/main/groovy/com/muwire/core/hostcache/CacheServers.groovy

@@ -8,10 +8,8 @@ class CacheServers {
     private static Set<Destination> CACHES = [
             // zlatinb
             new Destination("Wddh2E6FyyXBF7SvUYHKdN-vjf3~N6uqQWNeBDTM0P33YjiQCOsyedrjmDZmWFrXUJfJLWnCb5bnKezfk4uDaMyj~uvDG~yvLVcFgcPWSUd7BfGgym-zqcG1q1DcM8vfun-US7YamBlmtC6MZ2j-~Igqzmgshita8aLPCfNAA6S6e2UMjjtG7QIXlxpMec75dkHdJlVWbzrk9z8Qgru3YIk0UztYgEwDNBbm9wInsbHhr3HtAfa02QcgRVqRN2PnQXuqUJs7R7~09FZPEviiIcUpkY3FeyLlX1sgQFBeGeA96blaPvZNGd6KnNdgfLgMebx5SSxC-N4KZMSMBz5cgonQF3~m2HHFRSI85zqZNG5X9bJN85t80ltiv1W1es8ZnQW4es11r7MrvJNXz5bmSH641yJIvS6qI8OJJNpFVBIQSXLD-96TayrLQPaYw~uNZ-eXaE6G5dYhiuN8xHsFI1QkdaUaVZnvDGfsRbpS5GtpUbBDbyLkdPurG0i7dN1wAAAA"),
-            // sNL
-            new Destination("JC63wJNOqSJmymkj4~UJWywBTvDGikKMoYP0HX2Wz9c5l3otXSkwnxWAFL4cKr~Ygh3BNNi2t93vuLIiI1W8AsE42kR~PwRx~Y-WvIHXR6KUejRmOp-n8WidtjKg9k4aDy428uSOedqXDxys5mpoeQXwDsv1CoPTTwnmb1GWFy~oTGIsCguCl~aJWGnqiKarPO3GJQ~ev-NbvAQzUfC3HeP1e6pdI5CGGjExahTCID5UjpJw8GaDXWlGmYWWH303Xu4x-vAHQy1dJLsOBCn8dZravsn5BKJk~j0POUon45CCx-~NYtaPe0Itt9cMdD2ciC76Rep1D0X0sm1SjlSs8sZ52KmF3oaLZ6OzgI9QLMIyBUrfi41sK5I0qTuUVBAkvW1xr~L-20dYJ9TrbOaOb2-vDIfKaxVi6xQOuhgQDiSBhd3qv2m0xGu-BM9DQYfNA0FdMjnZmqjmji9RMavzQSsVFIbQGLbrLepiEFlb7TseCK5UtRp8TxnG7L4gbYevBQAEAAcAAA=="),
-            // dark_trion
-            new Destination("Gec9L29FVcQvYDgpcYuEYdltJn06PPoOWAcAM8Af-gDm~ehlrJcwlLXXs0hidq~yP2A0X7QcDi6i6shAfuEofTchxGJl8LRNqj9lio7WnB7cIixXWL~uCkD7Np5LMX0~akNX34oOb9RcBYVT2U5rFGJmJ7OtBv~IBkGeLhsMrqaCjahd0jdBO~QJ-t82ZKZhh044d24~JEfF9zSJxdBoCdAcXzryGNy7sYtFVDFsPKJudAxSW-UsSQiGw2~k-TxyF0r-iAt1IdzfNu8Lu0WPqLdhDYJWcPldx2PR5uJorI~zo~z3I5RX3NwzarlbD4nEP5s65ahPSfVCEkzmaJUBgP8DvBqlFaX89K4nGRYc7jkEjJ8cX4L6YPXUpTPWcfKkW259WdQY3YFh6x7rzijrGZewpczOLCrt-bZRYgDrUibmZxKZmNhy~lQu4gYVVjkz1i4tL~DWlhIc4y0x2vItwkYLArPPi~ejTnt-~Lhb7oPMXRcWa3UrwGKpFvGZY4NXBQAEAAcAAA==")
+            // echelon
+            new Destination("2MJTl8gYVPK43iJZJa~-5K1OchgPaPHXpqZmKIiKFvxyy8BlIJzUSrF4mazdta--shFHISfT0PEeI95j1yDyKMpGxatUyjSt3ZnyTfAehQR-H2kYV9FvjHo68uA9X5AaGYHKRYLuWMkihMXygd8ywoLjZtFP0UbKMPggfOZaWmjHF4081XoUXt~7MEAeYSQowndiUx0AH3HxNEiv0N373JJS61OsIXb5ctqVKkwIiX1R0ZxESzpP9Xwp8-T0ou8fsLksygbKyH~3K1CyTHjTS51Ux-U-CjOPH9rtCOjjAaifdyMpK0PxW1fVdoGswFywTz9Q-6DUMsIu5TsPMF0-UO1Wn8vCpVAWbBJAOtKCfBrGzp-E~GCbfCNs5xY19nLobMD5ehjsBdI1lXwGDCQ7kBOwC58uuC3BOoazgrB6IrGskyMTexawtthO9mhuPm91bq4xhNaCYHAe059xg5emnM7jFBVzQgjaZ5lOLn~HqcWofJ7oc0doE6XI6kOo~YncBQAEAAcAAA==")
         ]
 
     static List<Destination> getCacheServers() {

core/src/main/groovy/com/muwire/core/mesh/Mesh.groovy

@@ -10,7 +10,7 @@ import net.i2p.util.ConcurrentHashSet
 class Mesh {
     private final InfoHash infoHash
     private final Set<Persona> sources = new ConcurrentHashSet<>()
-    private final Pieces pieces
+    final Pieces pieces
 
     Mesh(InfoHash infoHash, Pieces pieces) {
         this.infoHash = infoHash

core/src/main/groovy/com/muwire/core/search/ResultsSender.groovy

@@ -88,7 +88,8 @@ class ResultsSender {
                     sources : suggested,
                     comment : comment,
                     certificates : certificates,
-                    chat : chatServer.running.get() && settings.advertiseChat
+                    chat : chatServer.running.get() && settings.advertiseChat,
+                    feed : settings.fileFeed && settings.advertiseFeed
                     )
                 uiResultEvents << uiResultEvent
             }
@@ -138,6 +139,8 @@ class ResultsSender {
                         os.write("Count: $results.length\r\n".getBytes(StandardCharsets.US_ASCII))
                         boolean chat = chatServer.running.get() && settings.advertiseChat
                         os.write("Chat: $chat\r\n".getBytes(StandardCharsets.US_ASCII))
+                        boolean feed = settings.fileFeed && settings.advertiseFeed
+                        os.write("Feed: $feed\r\n".getBytes(StandardCharsets.US_ASCII))
                         os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
                         DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
                         results.each { 

core/src/main/groovy/com/muwire/core/search/UIResultEvent.groovy

@@ -18,7 +18,8 @@ class UIResultEvent extends Event {
     boolean browse
     int certificates
     boolean chat
-
+    boolean feed
+    
     @Override
     public String toString() {
         super.toString() + "name:$name size:$size sender:${sender.getHumanReadableName()} pieceSize $pieceSize"

core/src/main/groovy/com/muwire/core/tracker/TrackerResponder.groovy

@@ -0,0 +1,214 @@
+package com.muwire.core.tracker
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.logging.Level
+import java.util.stream.Collectors
+
+import com.muwire.core.Constants
+import com.muwire.core.InfoHash
+import com.muwire.core.MuWireSettings
+import com.muwire.core.Persona
+import com.muwire.core.download.DownloadManager
+import com.muwire.core.download.Pieces
+import com.muwire.core.files.FileManager
+import com.muwire.core.mesh.Mesh
+import com.muwire.core.mesh.MeshManager
+import com.muwire.core.trust.TrustLevel
+import com.muwire.core.trust.TrustService
+import com.muwire.core.util.DataUtil
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+import net.i2p.client.I2PSession
+import net.i2p.client.I2PSessionMuxedListener
+import net.i2p.client.SendMessageOptions
+import net.i2p.client.datagram.I2PDatagramDissector
+import net.i2p.client.datagram.I2PDatagramMaker
+import net.i2p.data.Base64
+
+@Log
+class TrackerResponder {
+    private final I2PSession i2pSession
+    private final MuWireSettings muSettings
+    private final FileManager fileManager
+    private final DownloadManager downloadManager
+    private final MeshManager meshManager
+    private final TrustService trustService
+    private final Persona me
+    
+    private final Map<UUID,Long> uuids = new HashMap<>()
+    private final Timer expireTimer = new Timer("tracker-responder-timer", true)
+    
+    private static final long UUID_LIFETIME = 10 * 60 * 1000
+    
+    TrackerResponder(I2PSession i2pSession, MuWireSettings muSettings,
+        FileManager fileManager, DownloadManager downloadManager,
+        MeshManager meshManager, TrustService trustService,
+        Persona me) {
+        this.i2pSession = i2pSession
+        this.muSettings = muSettings
+        this.fileManager = fileManager
+        this.downloadManager = downloadManager
+        this.meshManager = meshManager
+        this.trustService = trustService
+        this.me = me
+    }
+    
+    void start() {
+        i2pSession.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, Constants.TRACKER_PORT)
+        expireTimer.schedule({expireUUIDs()} as TimerTask, UUID_LIFETIME, UUID_LIFETIME)
+    }
+    
+    void stop() {
+        expireTimer.cancel()
+    }
+    
+    private void expireUUIDs() {
+        final long now = System.currentTimeMillis()
+        synchronized(uuids) {
+            for (Iterator<UUID> iter = uuids.keySet().iterator(); iter.hasNext();) {
+                UUID uuid = iter.next();
+                Long time = uuids.get(uuid)
+                if (now - time > UUID_LIFETIME)
+                    iter.remove()
+            }
+        }
+    }
+    
+    private void respond(host, json) {
+        log.info("responding to host $host with json $json")
+        
+        def message = JsonOutput.toJson(json)
+        def maker = new I2PDatagramMaker(i2pSession)
+        message = maker.makeI2PDatagram(message.bytes)
+        def options = new SendMessageOptions()
+        options.setSendLeaseSet(false)
+        i2pSession.sendMessage(host, message, 0, message.length, I2PSession.PROTO_DATAGRAM, Constants.TRACKER_PORT, Constants.TRACKER_PORT, options)
+    }
+    
+    class Listener implements I2PSessionMuxedListener {
+
+        @Override
+        public void messageAvailable(I2PSession session, int msgId, long size) {
+        }
+
+        @Override
+        public void messageAvailable(I2PSession session, int msgId, long size, int proto, int fromport, int toport) {
+            if (proto != I2PSession.PROTO_DATAGRAM) {
+                log.warning "Received unexpected protocol $proto"
+                return
+            }
+            
+            byte[] payload = session.receiveMessage(msgId)
+            def dissector = new I2PDatagramDissector()
+            try {
+                dissector.loadI2PDatagram(payload)
+                def sender = dissector.getSender()
+                
+                log.info("got a tracker datagram from ${sender.toBase32()}")
+                
+                // if not trusted, just drop it
+                TrustLevel trustLevel = trustService.getLevel(sender)
+                
+                if (trustLevel == TrustLevel.DISTRUSTED || 
+                    (trustLevel == TrustLevel.NEUTRAL && !muSettings.allowUntrusted)) {
+                    log.info("dropping, untrusted")
+                    return
+                }
+                
+                payload = dissector.getPayload()
+                def slurper = new JsonSlurper()
+                def json = slurper.parse(payload)
+                
+                if (json.type != "TrackerPing") {
+                    log.warning("unknown type $json.type")
+                    return
+                }
+                
+                def response = [:]
+                response.type = "TrackerPong"
+                response.me = me.toBase64()
+
+                if (json.infoHash == null) {
+                    log.warning("infoHash missing")
+                    return
+                }
+                
+                if (json.uuid == null) {
+                    log.warning("uuid missing")
+                    return
+                }
+                
+                UUID uuid = UUID.fromString(json.uuid)
+                synchronized(uuids) {
+                    if (uuids.containsKey(uuid)) {
+                        log.warning("duplicate uuid $uuid")
+                        return
+                    }
+                    uuids.put(uuid, System.currentTimeMillis())
+                }
+                response.uuid = json.uuid
+                                
+                if (!muSettings.allowTracking) {
+                    response.code = 403
+                    respond(sender, response)
+                    return
+                }
+                
+                if (json.version != 1) {
+                    log.warning("unknown version $json.version")
+                    response.code = 400
+                    response.message = "I only support version 1"
+                    respond(sender,response)
+                    return
+                }
+                
+                byte[] infoHashBytes = Base64.decode(json.infoHash)
+                InfoHash infoHash = new InfoHash(infoHashBytes)
+                
+                log.info("servicing request for infoHash ${json.infoHash} with uuid ${json.uuid}")
+                
+                if (!(fileManager.isShared(infoHash) || downloadManager.isDownloading(infoHash))) {
+                    response.code = 404
+                    respond(sender, response)
+                    return
+                }
+
+                Mesh mesh = meshManager.get(infoHash)   
+                  
+                if (fileManager.isShared(infoHash))
+                    response.code = 200
+                else if (mesh != null) {
+                    response.code = 206
+                    Pieces pieces = mesh.getPieces()
+                    response.xHave = DataUtil.encodeXHave(pieces, pieces.getnPieces())
+                }
+                
+                if (mesh != null)
+                    response.altlocs = mesh.getRandom(10, me).stream().map({it.toBase64()}).collect(Collectors.toList())
+                    
+                respond(sender,response)
+            } catch (Exception e) {
+                log.log(Level.WARNING, "invalid datagram", e)
+            }
+        }
+
+        @Override
+        public void reportAbuse(I2PSession session, int severity) {
+        }
+
+        @Override
+        public void disconnected(I2PSession session) {
+            log.severe("session disconnected")
+        }
+
+        @Override
+        public void errorOccurred(I2PSession session, String message, Throwable error) {
+            log.log(Level.SEVERE, message, error)
+        }
+        
+    }
+    
+    
+}

core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy

@@ -2,6 +2,7 @@ package com.muwire.core.update
 
 import java.util.logging.Level
 
+import com.muwire.core.Constants
 import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
@@ -63,7 +64,7 @@ class UpdateClient {
     }
 
     void start() {
-        session.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, 2)
+        session.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, Constants.UPDATE_PORT)
         timer.schedule({checkUpdate()} as TimerTask, 60000, 60 * 60 * 1000)
     }
 
@@ -108,7 +109,7 @@ class UpdateClient {
         ping = maker.makeI2PDatagram(ping.bytes)
         def options = new SendMessageOptions()
         options.setSendLeaseSet(true)
-        session.sendMessage(UpdateServers.UPDATE_SERVER, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, 2, 0, options)
+        session.sendMessage(UpdateServers.UPDATE_SERVER, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, Constants.UPDATE_PORT, 0, options)
     }
 
     class Listener implements I2PSessionMuxedListener {

core/src/main/groovy/com/muwire/core/upload/ContentUploader.groovy

@@ -106,7 +106,7 @@ class ContentUploader extends Uploader {
             return done ? 100 : 0
         int position = mapped.position()
         int total = request.getRange().end - request.getRange().start
-        (int)(position * 100.0 / total)
+        (int)(position * 100.0d / total)
     }
 
     @Override

core/src/main/groovy/com/muwire/core/upload/HashListUploader.groovy

@@ -45,7 +45,7 @@ class HashListUploader extends Uploader {
 
     @Override
     public synchronized int getProgress() {
-        (int)(mapped.position() * 100.0 / mapped.capacity())
+        (int)(mapped.position() * 100.0d / mapped.capacity())
     }
 
     @Override

core/src/main/groovy/com/muwire/core/upload/Request.groovy

@@ -22,7 +22,7 @@ class Request {
 
     static Request parseContentRequest(InfoHash infoHash, InputStream is) throws IOException {
 
-        Map<String, String> headers = parseHeaders(is)
+        Map<String, String> headers = DataUtil.readAllHeaders(is)
 
         if (!headers.containsKey("Range"))
             throw new IOException("Range header not found")
@@ -60,7 +60,7 @@ class Request {
     }
 
     static Request parseHashListRequest(InfoHash infoHash, InputStream is) throws IOException {
-        Map<String,String> headers = parseHeaders(is)
+        Map<String,String> headers = DataUtil.readAllHeaders(is)
         Persona downloader = null
         if (headers.containsKey("X-Persona")) {
             def encoded = headers["X-Persona"].trim()
@@ -69,55 +69,4 @@ class Request {
         }
         new HashListRequest(infoHash : infoHash, headers : headers, downloader : downloader)
     }
-
-    private static Map<String, String> parseHeaders(InputStream is) {
-        Map<String,String> headers = new HashMap<>()
-        byte [] tmp = new byte[Constants.MAX_HEADER_SIZE]
-        while(headers.size() < Constants.MAX_HEADERS) {
-            boolean r = false
-            boolean n = false
-            int idx = 0
-            while (true) {
-                byte read = is.read()
-                if (read == -1)
-                    throw new IOException("Stream closed")
-
-                if (!r && read == N)
-                    throw new IOException("Received N before R")
-                if (read == R) {
-                    if (r)
-                        throw new IOException("double R")
-                    r = true
-                    continue
-                }
-
-                if (r && !n) {
-                    if (read != N)
-                        throw new IOException("R not followed by N")
-                    n = true
-                    break
-                }
-                if (idx == 0x1 << 14)
-                    throw new IOException("Header too long")
-                tmp[idx++] = read
-            }
-
-            if (idx == 0)
-                break
-
-            String header = new String(tmp, 0, idx, StandardCharsets.US_ASCII)
-            log.fine("Read header $header")
-
-            int keyIdx = header.indexOf(":")
-            if (keyIdx < 1)
-                throw new IOException("Header key not found")
-            if (keyIdx == header.length())
-                throw new IOException("Header value not found")
-            String key = header.substring(0, keyIdx)
-            String value = header.substring(keyIdx + 1)
-            headers.put(key, value)
-        }
-        headers
-    }
-
 }

core/src/main/groovy/com/muwire/core/upload/Uploader.groovy

@@ -49,7 +49,7 @@ abstract class Uploader {
         final long now = System.currentTimeMillis()
         long interval = Math.max(1000, now - lastSpeedRead)
         lastSpeedRead = now;
-        int currSpeed = (int) (dataSinceLastRead * 1000.0 / interval)
+        int currSpeed = (int) (dataSinceLastRead * 1000.0d / interval)
         dataSinceLastRead = 0
 
         // normalize to speedArr.size

core/src/main/java/com/muwire/core/Constants.java

@@ -4,6 +4,7 @@ import net.i2p.crypto.SigType;
 
 public class Constants {
     public static final byte PERSONA_VERSION = (byte)1;
+    public static final String INVALID_NICKNAME_CHARS = "'\"();<>=@$%";
     public static final byte FILE_CERT_VERSION = (byte)2;
     public static final int CHAT_VERSION = 1;
     
@@ -17,5 +18,8 @@ public class Constants {
     
     public static final int MAX_COMMENT_LENGTH = 0x1 << 15;
     
-    public static final long MAX_QUERY_AGE = 5 * 60 * 1000L; 
+    public static final long MAX_QUERY_AGE = 5 * 60 * 1000L;
+    
+    public static final int UPDATE_PORT = 2;
+    public static final int TRACKER_PORT = 3;
 }

core/src/main/java/com/muwire/core/InvalidNicknameException.java

@@ -0,0 +1,25 @@
+package com.muwire.core;
+
+public class InvalidNicknameException extends Exception {
+
+    public InvalidNicknameException() {
+    }
+
+    public InvalidNicknameException(String message) {
+        super(message);
+    }
+
+    public InvalidNicknameException(Throwable cause) {
+        super(cause);
+    }
+
+    public InvalidNicknameException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidNicknameException(String message, Throwable cause, boolean enableSuppression,
+            boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+}

core/src/main/java/com/muwire/core/Persona.java

@@ -7,6 +7,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import com.muwire.core.util.DataUtil;
+
 import net.i2p.crypto.DSAEngine;
 import net.i2p.data.Base64;
 import net.i2p.data.DataFormatException;
@@ -25,12 +27,15 @@ public class Persona {
     private volatile String base64;
     private volatile byte[] payload;
 
-    public Persona(InputStream personaStream) throws IOException, DataFormatException, InvalidSignatureException {
+    public Persona(InputStream personaStream) throws IOException, DataFormatException, InvalidSignatureException, InvalidNicknameException {
         version = (byte) (personaStream.read() & 0xFF);
         if (version != Constants.PERSONA_VERSION)
             throw new IOException("Unknown version "+version);
 
         name = new Name(personaStream);
+        if (!DataUtil.isValidName(name.name))
+            throw new InvalidNicknameException(name.name + " is not a valid nickname");
+        
         destination = Destination.create(personaStream);
         sig = new byte[SIG_LEN];
         DataInputStream dis = new DataInputStream(personaStream);
@@ -38,7 +43,7 @@ public class Persona {
         if (!verify(version, name, destination, sig))
             throw new InvalidSignatureException(getHumanReadableName() + " didn't verify");
     }
-
+    
     private static boolean verify(byte version, Name name, Destination destination, byte [] sig) 
     throws IOException, DataFormatException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();

core/src/main/java/com/muwire/core/SharedFile.java

@@ -31,6 +31,8 @@ public class SharedFile {
     private volatile String comment;
     private final Set<String> downloaders = Collections.synchronizedSet(new HashSet<>());
     private final Set<SearchEntry> searches = Collections.synchronizedSet(new HashSet<>());
+    private volatile boolean published;
+    private volatile long publishedTimestamp;
 
     public SharedFile(File file, byte[] root, int pieceSize) throws IOException {
         this.file = file;
@@ -114,6 +116,24 @@ public class SharedFile {
     public void addDownloader(String name) {
         downloaders.add(name);
     }
+    
+    public void publish(long timestamp) {
+        published = true;
+        publishedTimestamp = timestamp;
+    }
+    
+    public void unpublish() {
+        published = false;
+        publishedTimestamp = 0;
+    }
+    
+    public boolean isPublished() {
+        return published;
+    }
+    
+    public long getPublishedTimestamp() {
+        return publishedTimestamp;
+    }
 
     @Override
     public int hashCode() {
@@ -139,6 +159,18 @@ public class SharedFile {
             this.query = query;
         }
         
+        public Persona getSearcher() {
+            return searcher;
+        }
+        
+        public long getTimestamp() {
+            return timestamp;
+        }
+        
+        public String getQuery() {
+            return query;
+        }
+        
         public int hashCode() {
             return Objects.hash(searcher) ^ Objects.hash(timestamp) ^ query.hashCode();
         }

core/src/main/java/com/muwire/core/filefeeds/Feed.java

@@ -0,0 +1,81 @@
+package com.muwire.core.filefeeds;
+
+import com.muwire.core.Persona;
+
+public class Feed {
+    
+    private final Persona publisher;
+    
+    private int updateInterval;
+    private long lastUpdated;
+    private volatile long lastUpdateAttempt;
+    private int itemsToKeep;
+    private boolean autoDownload;
+    private boolean sequential;
+    private FeedFetchStatus status;
+    
+    public Feed(Persona publisher) {
+        this.publisher = publisher;
+        this.status = FeedFetchStatus.IDLE;
+    }
+
+    public int getUpdateInterval() {
+        return updateInterval;
+    }
+
+    public void setUpdateInterval(int updateInterval) {
+        this.updateInterval = updateInterval;
+    }
+
+    public long getLastUpdated() {
+        return lastUpdated;
+    }
+
+    public void setLastUpdated(long lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public int getItemsToKeep() {
+        return itemsToKeep;
+    }
+
+    public void setItemsToKeep(int itemsToKeep) {
+        this.itemsToKeep = itemsToKeep;
+    }
+
+    public boolean isAutoDownload() {
+        return autoDownload;
+    }
+
+    public void setAutoDownload(boolean autoDownload) {
+        this.autoDownload = autoDownload;
+    }
+
+    public Persona getPublisher() {
+        return publisher;
+    }
+
+    public void setStatus(FeedFetchStatus status) {
+        this.status = status;
+    }
+    
+    public FeedFetchStatus getStatus() {
+        return status;
+    }
+    
+    public void setSequential(boolean sequential) {
+        this.sequential = sequential;
+    }
+    
+    public boolean isSequential() {
+        return sequential;
+    }
+    
+    public void setLastUpdateAttempt(long lastUpdateAttempt) {
+        this.lastUpdateAttempt = lastUpdateAttempt;
+    }
+    
+    public long getLastUpdateAttempt() {
+        return lastUpdateAttempt;
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/FeedFetchStatus.java

@@ -0,0 +1,19 @@
+package com.muwire.core.filefeeds;
+
+public enum FeedFetchStatus {
+    IDLE(false), 
+    CONNECTING(true), 
+    FETCHING(true), 
+    FINISHED(false), 
+    FAILED(false);
+    
+    private final boolean active;
+    
+    FeedFetchStatus(boolean active) {
+        this.active = active;
+    }
+    
+    public boolean isActive() {
+        return active;
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/FeedItem.java

@@ -0,0 +1,79 @@
+package com.muwire.core.filefeeds;
+
+import java.util.Objects;
+
+import com.muwire.core.InfoHash;
+import com.muwire.core.Persona;
+
+public class FeedItem {
+
+    private final Persona publisher;
+    private final long timestamp;
+    private final String name;
+    private final long size;
+    private final int pieceSize;
+    private final InfoHash infoHash;
+    private final int certificates;
+    private final String comment;
+    
+    public FeedItem(Persona publisher, long timestamp, String name, long size, int pieceSize, InfoHash infoHash,
+            int certificates, String comment) {
+        super();
+        this.publisher = publisher;
+        this.timestamp = timestamp;
+        this.name = name;
+        this.size = size;
+        this.pieceSize = pieceSize;
+        this.infoHash = infoHash;
+        this.certificates = certificates;
+        this.comment = comment;
+    }
+
+    public Persona getPublisher() {
+        return publisher;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public int getPieceSize() {
+        return pieceSize;
+    }
+
+    public InfoHash getInfoHash() {
+        return infoHash;
+    }
+
+    public int getCertificates() {
+        return certificates;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(publisher, timestamp, name, infoHash);
+    }
+    
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof FeedItem))
+            return false;
+        FeedItem other = (FeedItem)o;
+        return Objects.equals(publisher, other.publisher) &&
+                timestamp == other.timestamp &&
+                Objects.equals(name, other.name) &&
+                Objects.equals(infoHash, other.infoHash);
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/InvalidFeedItemException.java

@@ -0,0 +1,30 @@
+package com.muwire.core.filefeeds;
+
+public class InvalidFeedItemException extends Exception {
+
+    public InvalidFeedItemException() {
+        super();
+    }
+
+    public InvalidFeedItemException(String message, Throwable cause, boolean enableSuppression,
+            boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(String message, Throwable cause) {
+        super(message, cause);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(String message) {
+        super(message);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(Throwable cause) {
+        super(cause);
+        // TODO Auto-generated constructor stub
+    }
+
+}

core/src/main/java/com/muwire/core/util/DataUtil.java

@@ -58,9 +58,9 @@ public class DataUtil {
         if (header.length != 3)
             throw new IllegalArgumentException("header length $header.length");
 
-        return (((int)(header[0] & 0x7F)) << 16) |
-                (((int)(header[1] & 0xFF) << 8)) |
-                ((int)header[2] & 0xFF);
+        return ((header[0] & 0x7F) << 16) |
+                ((header[1] & 0xFF) << 8) |
+                (header[2] & 0xFF);
     }
 
     public static String readi18nString(byte [] encoded) {
@@ -174,7 +174,7 @@ public class DataUtil {
                 clean.setAccessible(true);
                 clean.invoke(cleaner.invoke(cb));
             } else {
-                Class unsafeClass;
+                Class<?> unsafeClass;
                 try {
                     unsafeClass = Class.forName("sun.misc.Unsafe");
                 } catch(Exception ex) {
@@ -216,4 +216,11 @@ public class DataUtil {
         Signature sig = DSAEngine.getInstance().sign(payload, spk);
         return sig.getData();
     }
+    
+    public static boolean isValidName(String name) {
+        for (int i = 0; i < Constants.INVALID_NICKNAME_CHARS.length(); i++)
+            if (name.indexOf(Constants.INVALID_NICKNAME_CHARS.charAt(i)) >= 0)
+                return false;
+        return true;
+    }
 }

core/src/test/groovy/com/muwire/core/files/FileManagerTest.groovy

@@ -39,13 +39,13 @@ class FileManagerTest {
     @Test
     void testHash1Result() {
         File f = new File("a b.c")
-        InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih, 0)
+        byte [] root = new byte[32]
+        SharedFile sf = new SharedFile(f,root, 0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
         UUID uuid = UUID.randomUUID()
-        SearchEvent se = new SearchEvent(searchHash: ih.getRoot(), uuid: uuid)
+        SearchEvent se = new SearchEvent(searchHash: root, uuid: uuid)
 
         manager.onSearchEvent(se)
         Thread.sleep(20)
@@ -58,14 +58,14 @@ class FileManagerTest {
 
     @Test
     void testHash2Results() {
-        InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(new File("a b.c"), ih, 0)
-        SharedFile sf2 = new SharedFile(new File("d e.f"), ih, 0)
+        byte [] root = new byte[32]
+        SharedFile sf1 = new SharedFile(new File("a b.c"), root, 0)
+        SharedFile sf2 = new SharedFile(new File("d e.f"), root, 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf1)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf2)
 
         UUID uuid = UUID.randomUUID()
-        SearchEvent se = new SearchEvent(searchHash: ih.getRoot(), uuid: uuid)
+        SearchEvent se = new SearchEvent(searchHash: root, uuid: uuid)
 
         manager.onSearchEvent(se)
         Thread.sleep(20)
@@ -81,7 +81,7 @@ class FileManagerTest {
     void testHash0Results() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih, 0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(), 0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -95,7 +95,7 @@ class FileManagerTest {
     void testKeyword1Result() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih,0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(),0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -113,12 +113,12 @@ class FileManagerTest {
     void testKeyword2Results() {
         File f1 = new File("a b.c")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf1)
 
         File f2 = new File("c d.e")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf2)
 
         UUID uuid = UUID.randomUUID()
@@ -136,7 +136,7 @@ class FileManagerTest {
     void testKeyword0Results() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih,0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(),0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -149,8 +149,8 @@ class FileManagerTest {
     @Test
     void testRemoveFileExistingHash() {
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(new File("a b.c"), ih, 0)
-        SharedFile sf2 = new SharedFile(new File("d e.f"), ih, 0)
+        SharedFile sf1 = new SharedFile(new File("a b.c"), ih.getRoot(), 0)
+        SharedFile sf2 = new SharedFile(new File("d e.f"), ih.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf1)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf2)
 
@@ -167,12 +167,12 @@ class FileManagerTest {
     void testRemoveFile() {
         File f1 = new File("a b.c")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf1)
 
         File f2 = new File("c d.e")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf2)
 
         manager.onFileUnsharedEvent new FileUnsharedEvent(deleted : true, unsharedFile: sf2)
@@ -198,7 +198,7 @@ class FileManagerTest {
         comment = Base64.encode(DataUtil.encodei18nString(comment))
         File f1 = new File("MuWire-0.5.10.AppImage")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         sf1.setComment(comment)
         
         manager.onFileLoadedEvent(new FileLoadedEvent(loadedFile : sf1))
@@ -206,7 +206,7 @@ class FileManagerTest {
         
         File f2 = new File("MuWire-0.6.0.AppImage")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         sf2.setComment(comment)
         
         manager.onFileLoadedEvent(new FileLoadedEvent(loadedFile : sf2))

core/src/test/groovy/com/muwire/core/files/HasherServiceTest.groovy

@@ -45,7 +45,7 @@ class HasherServiceTest {
         def hashed = listener.poll()
         assert hashed instanceof FileHashedEvent
         assert hashed.sharedFile.file == f.getCanonicalFile()
-        assert hashed.sharedFile.infoHash != null
+        assert hashed.sharedFile.root != null
         assert listener.isEmpty()
     }
 

core/src/test/groovy/com/muwire/core/files/PersisterServiceLoadingTest.groovy

@@ -85,7 +85,7 @@ class PersisterServiceLoadingTest {
         def loadedFile = listener.publishedFiles[0]
         assert loadedFile != null
         assert loadedFile.file == sharedFile1.getCanonicalFile()
-        assert loadedFile.infoHash == ih1
+        assert loadedFile.root == ih1.getRoot()
     }
 
     private static String getSharedFileJsonName(File sharedFile) {
@@ -128,7 +128,7 @@ class PersisterServiceLoadingTest {
         def loadedFile = listener.publishedFiles[0]
         assert loadedFile != null
         assert loadedFile.file == sharedFile1.getCanonicalFile()
-        assert loadedFile.infoHash == ih1
+        assert loadedFile.root == ih1.getRoot()
     }
 
     @Test
@@ -169,10 +169,10 @@ class PersisterServiceLoadingTest {
         assert listener.publishedFiles.size() == 2
         def loadedFile1 = listener.publishedFiles[0]
         assert loadedFile1.file == sharedFile1.getCanonicalFile()
-        assert loadedFile1.infoHash == ih1
+        assert loadedFile1.root == ih1.getRoot()
         def loadedFile2 = listener.publishedFiles[1]
         assert loadedFile2.file == sharedFile2.getCanonicalFile()
-        assert loadedFile2.infoHash == ih2
+        assert loadedFile2.root == ih2.getRoot()
     }
 
     @Test

core/src/test/groovy/com/muwire/core/files/PersisterServiceSavingTest.groovy

@@ -2,6 +2,7 @@ package com.muwire.core.files
 
 import org.junit.After
 import org.junit.Before
+import org.junit.Ignore
 import org.junit.Test
 
 import com.muwire.core.Destinations
@@ -16,6 +17,7 @@ import groovy.json.JsonSlurper
 import net.i2p.data.Base32
 import net.i2p.data.Base64
 
+@Ignore
 class PersisterServiceSavingTest {
 
     File f

gradle.properties

@@ -1,6 +1,6 @@
 group         = com.muwire
-version       = 0.6.10
-i2pVersion    = 0.9.44
+version       = 0.6.14
+i2pVersion    = 0.9.45
 groovyVersion = 2.4.15
 slf4jVersion  = 1.7.25
 spockVersion  = 1.1-groovy-2.4
@@ -8,8 +8,10 @@ grailsVersion=4.0.0
 gorm.version=7.0.2.RELEASE
 griffonEnv=prod
 
+# javac properties
 sourceCompatibility=1.8
 targetCompatibility=1.8
+compilerArgs=-Xlint:unchecked,cast,path,divzero,empty,path,finally,overrides
 
 # plugin properties
 author = zab@mail.i2p

gui/griffon-app/conf/Config.groovy

@@ -126,4 +126,19 @@ mvcGroups {
         view = 'com.muwire.gui.ChatMonitorView'
         controller = 'com.muwire.gui.ChatMonitorController'
     }
+    'feed-configuration' {
+        model = 'com.muwire.gui.FeedConfigurationModel'
+        view = 'com.muwire.gui.FeedConfigurationView'
+        controller = 'com.muwire.gui.FeedConfigurationController'
+    }
+    'watched-directory' {
+        model = 'com.muwire.gui.WatchedDirectoryModel'
+        view = 'com.muwire.gui.WatchedDirectoryView'
+        controller = 'com.muwire.gui.WatchedDirectoryController'
+    }
+    'sign' {
+        model = 'com.muwire.gui.SignModel'
+        view = 'com.muwire.gui.SignView'
+        controller = 'com.muwire.gui.SignController'
+    }
 }

gui/griffon-app/controllers/com/muwire/gui/AdvancedSharingController.groovy

@@ -7,6 +7,7 @@ import griffon.metadata.ArtifactProviderFor
 import javax.annotation.Nonnull
 
 import com.muwire.core.Core
+import com.muwire.core.files.directories.UISyncDirectoryEvent
 
 @ArtifactProviderFor(GriffonController)
 class AdvancedSharingController {
@@ -14,4 +15,25 @@ class AdvancedSharingController {
     AdvancedSharingModel model
     @MVCMember @Nonnull
     AdvancedSharingView view
+    
+    @ControllerAction
+    void configure() {
+        def wd = view.selectedWatchedDirectory()
+        if (wd == null)
+            return
+            
+        def params = [:]
+        params['core'] = model.core
+        params['directory'] = wd
+        mvcGroup.createMVCGroup("watched-directory",params)
+    }
+    
+    @ControllerAction
+    void sync() {
+        def wd = view.selectedWatchedDirectory()
+        if (wd == null)
+            return
+        def event = new UISyncDirectoryEvent(directory : wd.directory)
+        model.core.eventBus.publish(event)
+    }
 }

gui/griffon-app/controllers/com/muwire/gui/BrowseController.groovy

@@ -113,7 +113,9 @@ class BrowseController {
             return
         
         def params = [:]
-        params['result'] = result
+        params['host'] = result.getSender()
+        params['infoHash'] = result.getInfohash()
+        params['name'] = result.getName()
         params['core'] = core
         mvcGroup.createMVCGroup("fetch-certificates", params)
     }

gui/griffon-app/controllers/com/muwire/gui/FeedConfigurationController.groovy

@@ -0,0 +1,36 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import javax.annotation.Nonnull
+
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
+
+@ArtifactProviderFor(GriffonController)
+class FeedConfigurationController {
+    @MVCMember @Nonnull
+    FeedConfigurationModel model
+    @MVCMember @Nonnull
+    FeedConfigurationView view
+
+    @ControllerAction
+    void save() {
+        
+        model.feed.setAutoDownload(view.autoDownloadCheckbox.model.isSelected())
+        model.feed.setSequential(view.sequentialCheckbox.model.isSelected())
+        model.feed.setItemsToKeep(Integer.parseInt(view.itemsToKeepField.text))
+        model.feed.setUpdateInterval(Integer.parseInt(view.updateIntervalField.text) * 60000)
+        
+        model.core.eventBus.publish(new UIFeedConfigurationEvent(feed : model.feed))
+        
+        cancel()    
+    }
+    
+    @ControllerAction
+    void cancel() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/controllers/com/muwire/gui/FetchCertificatesController.groovy

@@ -28,7 +28,7 @@ class FetchCertificatesController {
         core.eventBus.with { 
             register(CertificateFetchEvent.class, this)
             register(CertificateFetchedEvent.class, this)
-            publish(new UIFetchCertificatesEvent(host : model.result.sender, infoHash : model.result.infohash))
+            publish(new UIFetchCertificatesEvent(host : model.host, infoHash : model.infoHash))
         }
     }
     

gui/griffon-app/controllers/com/muwire/gui/MainFrameController.groovy

@@ -30,6 +30,13 @@ import com.muwire.core.download.UIDownloadCancelledEvent
 import com.muwire.core.download.UIDownloadPausedEvent
 import com.muwire.core.download.UIDownloadResumedEvent
 import com.muwire.core.filecert.UICreateCertificateEvent
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.FeedItem
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
+import com.muwire.core.filefeeds.UIFeedDeletedEvent
+import com.muwire.core.filefeeds.UIFeedUpdateEvent
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
 import com.muwire.core.files.FileUnsharedEvent
 import com.muwire.core.search.QueryEvent
 import com.muwire.core.search.SearchEvent
@@ -505,6 +512,105 @@ class MainFrameController {
         clipboard.setContents(selection, null)
     }
     
+    @ControllerAction
+    void publish() {
+        def selectedFiles = view.selectedSharedFiles()
+        if (selectedFiles == null || selectedFiles.isEmpty())
+            return
+        
+        if (model.publishButtonText == "Unpublish") {
+            selectedFiles.each { 
+                it.unpublish()
+                model.core.eventBus.publish(new UIFileUnpublishedEvent(sf : it))
+            }
+        } else {
+            long now = System.currentTimeMillis()
+            selectedFiles.stream().filter({!it.isPublished()}).forEach({
+                it.publish(now)
+                model.core.eventBus.publish(new UIFilePublishedEvent(sf : it))
+            })
+        }
+        view.refreshSharedFiles()
+    }
+    
+    @ControllerAction
+    void updateFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        model.core.eventBus.publish(new UIFeedUpdateEvent(host: feed.getPublisher()))
+    }
+    
+    @ControllerAction
+    void unsubscribeFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        model.core.eventBus.publish(new UIFeedDeletedEvent(host : feed.getPublisher()))
+        runInsideUIAsync {
+            model.feeds.remove(feed)
+            model.feedItems.clear()
+            view.refreshFeeds()
+        }
+    }
+    
+    @ControllerAction
+    void configureFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        
+        def params = [:]
+        params['core'] = core
+        params['feed'] = feed
+        mvcGroup.createMVCGroup("feed-configuration", params)
+    }
+    
+    @ControllerAction
+    void downloadFeedItem() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.isEmpty())
+            return
+        Feed f = model.core.getFeedManager().getFeed(items.get(0).getPublisher())
+        items.each { 
+            if (!model.canDownload(it.getInfoHash()))
+                return
+            File target = new File(application.context.get("muwire-settings").downloadLocation, it.getName())
+            model.core.eventBus.publish(new UIDownloadFeedItemEvent(item : it, target : target, sequential : f.isSequential()))
+        }
+        view.showDownloadsWindow.call()
+    }
+    
+    @ControllerAction
+    void viewFeedItemComment() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.size() != 1)
+            return
+        FeedItem item = items.get(0)
+        
+        String groupId = Base64.encode(item.getInfoHash().getRoot())
+        Map<String, Object> params = new HashMap<>()
+        params['text'] = DataUtil.readi18nString(Base64.decode(item.getComment()))
+        params['name'] = item.getName()
+        
+        mvcGroup.createMVCGroup("show-comment", groupId, params)
+    }
+    
+    @ControllerAction
+    void viewFeedItemCertificates() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.size() != 1)
+            return
+        FeedItem item = items.get(0)
+        
+        def params = [:]
+        params['core'] = core
+        params['host'] = item.getPublisher()
+        params['infoHash'] = item.getInfoHash()
+        params['name'] = item.getName()
+        mvcGroup.createMVCGroup("fetch-certificates", params)
+    }
+    
     void startChat(Persona p) {
         if (!mvcGroup.getChildrenGroups().containsKey(p.getHumanReadableName())) {
             def params = [:]

gui/griffon-app/controllers/com/muwire/gui/OptionsController.groovy

@@ -104,6 +104,10 @@ class OptionsController {
         model.browseFiles = browseFiles
         settings.browseFiles = browseFiles
         
+        boolean allowTracking = view.allowTrackingCheckbox.model.isSelected()
+        model.allowTracking = allowTracking
+        settings.allowTracking = allowTracking
+        
         text = view.speedSmoothSecondsField.text
         model.speedSmoothSeconds = Integer.valueOf(text)
         settings.speedSmoothSeconds = Integer.valueOf(text)
@@ -122,7 +126,38 @@ class OptionsController {
             model.outBw = text
             settings.outBw = Integer.valueOf(text)
         }
+        
+        // feed saving
+        
+        boolean fileFeed = view.fileFeedCheckbox.model.isSelected()
+        model.fileFeed = fileFeed
+        settings.fileFeed = fileFeed
+        
+        boolean advertiseFeed = view.advertiseFeedCheckbox.model.isSelected()
+        model.advertiseFeed = advertiseFeed
+        settings.advertiseFeed = advertiseFeed
+        
+        boolean autoPublishSharedFiles = view.autoPublishSharedFilesCheckbox.model.isSelected()
+        model.autoPublishSharedFiles = autoPublishSharedFiles
+        settings.autoPublishSharedFiles = autoPublishSharedFiles
+        
+        boolean defaultFeedAutoDownload = view.defaultFeedAutoDownloadCheckbox.model.isSelected()
+        model.defaultFeedAutoDownload = defaultFeedAutoDownload
+        settings.defaultFeedAutoDownload = defaultFeedAutoDownload
+        
+        boolean defaultFeedSequential = view.defaultFeedSequentialCheckbox.model.isSelected()
+        model.defaultFeedSequential = defaultFeedSequential
+        settings.defaultFeedSequential = defaultFeedSequential
+        
+        String defaultFeedItemsToKeep = view.defaultFeedItemsToKeepField.text
+        model.defaultFeedItemsToKeep = defaultFeedItemsToKeep
+        settings.defaultFeedItemsToKeep = Integer.parseInt(defaultFeedItemsToKeep)
+        
+        String defaultFeedUpdateInterval = view.defaultFeedUpdateIntervalField.text
+        model.defaultFeedUpdateInterval = defaultFeedUpdateInterval
+        settings.defaultFeedUpdateInterval = Integer.parseInt(defaultFeedUpdateInterval)
 
+        // trust saving
 
         boolean onlyTrusted = view.allowUntrustedCheckbox.model.isSelected()
         model.onlyTrusted = onlyTrusted

gui/griffon-app/controllers/com/muwire/gui/SearchTabController.groovy

@@ -12,6 +12,8 @@ import javax.swing.JOptionPane
 import com.muwire.core.Core
 import com.muwire.core.Persona
 import com.muwire.core.download.UIDownloadEvent
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
 import com.muwire.core.search.UIResultEvent
 import com.muwire.core.trust.TrustEvent
 import com.muwire.core.trust.TrustLevel
@@ -107,6 +109,22 @@ class SearchTabController {
         mvcGroup.createMVCGroup("browse", groupId, params)
     }
     
+    @ControllerAction
+    void subscribe() {
+        def sender = view.selectedSender()
+        if (sender == null)
+            return
+
+        Feed feed = new Feed(sender)
+        feed.setAutoDownload(core.muOptions.defaultFeedAutoDownload)
+        feed.setSequential(core.muOptions.defaultFeedSequential)
+        feed.setItemsToKeep(core.muOptions.defaultFeedItemsToKeep)
+        feed.setUpdateInterval(core.muOptions.defaultFeedUpdateInterval * 60 * 1000)
+        
+        core.eventBus.publish(new UIFeedConfigurationEvent(feed : feed, newFeed: true))  
+        mvcGroup.parentGroup.view.showFeedsWindow.call()          
+    }
+    
     @ControllerAction
     void chat() {
         def sender = view.selectedSender()
@@ -139,7 +157,9 @@ class SearchTabController {
             return
 
         def params = [:]
-        params['result'] = event
+        params['host'] = event.getSender()
+        params['infoHash'] = event.getInfohash()
+        params['name'] = event.getName()
         params['core'] = core
         mvcGroup.createMVCGroup("fetch-certificates", params)
     }

gui/griffon-app/controllers/com/muwire/gui/SignController.groovy

@@ -0,0 +1,50 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import net.i2p.crypto.DSAEngine
+import net.i2p.data.Base64
+
+import java.awt.Toolkit
+import java.awt.datatransfer.StringSelection
+import java.nio.charset.StandardCharsets
+
+import javax.annotation.Nonnull
+import javax.swing.JOptionPane
+
+import com.muwire.core.Constants
+import com.muwire.core.Core
+import com.muwire.core.util.DataUtil
+
+@ArtifactProviderFor(GriffonController)
+class SignController {
+    
+    Core core
+    
+    @MVCMember @Nonnull
+    SignView view
+
+    @ControllerAction
+    void sign() {
+        String plain = view.plainTextArea.getText()
+        byte[] payload = plain.getBytes(StandardCharsets.UTF_8)
+        def sig = DSAEngine.getInstance().sign(payload, core.spk)
+        view.signedTextArea.setText(Base64.encode(sig.data))
+    }
+    
+    @ControllerAction
+    void copy() {
+        String signed = view.signedTextArea.getText()
+        StringSelection selection = new StringSelection(signed)
+        def clipboard = Toolkit.getDefaultToolkit().getSystemClipboard()
+        clipboard.setContents(selection, null)
+    }
+    
+    @ControllerAction
+    void close() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/controllers/com/muwire/gui/WatchedDirectoryController.groovy

@@ -0,0 +1,33 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import javax.annotation.Nonnull
+
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+
+@ArtifactProviderFor(GriffonController)
+class WatchedDirectoryController {
+    @MVCMember @Nonnull
+    WatchedDirectoryModel model
+    @MVCMember @Nonnull
+    WatchedDirectoryView view
+
+    @ControllerAction
+    void save() {
+        def event = new WatchedDirectoryConfigurationEvent(
+            directory : model.directory.directory,
+            autoWatch : view.autoWatchCheckbox.model.isSelected(),
+            syncInterval : Integer.parseInt(view.syncIntervalField.text))
+        model.core.eventBus.publish(event)
+        cancel()
+    }
+    
+    @ControllerAction
+    void cancel() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/lifecycle/Ready.groovy

@@ -6,10 +6,12 @@ import net.i2p.util.SystemVersion
 
 import org.codehaus.griffon.runtime.core.AbstractLifecycleHandler
 
+import com.muwire.core.Constants
 import com.muwire.core.Core
 import com.muwire.core.MuWireSettings
 import com.muwire.core.UILoadedEvent
 import com.muwire.core.files.FileSharedEvent
+import com.muwire.core.util.DataUtil
 
 import javax.annotation.Nonnull
 import javax.inject.Inject
@@ -82,23 +84,23 @@ class Ready extends AbstractLifecycleHandler {
         Core core
         try {
             core = new Core(props, home, metadata["application.version"])
+            Runtime.getRuntime().addShutdownHook({
+                core.shutdown()
+            })
+            core.startServices()
+            application.context.put("muwire-settings", props)
+            application.context.put("core",core)
+            application.getPropertyChangeListeners("core").each {
+                it.propertyChange(new PropertyChangeEvent(this, "core", null, core))
+            }
+
+            core.eventBus.publish(new UILoadedEvent())
         } catch (Exception bad) {
             log.log(Level.SEVERE,"couldn't initialize core",bad)
             JOptionPane.showMessageDialog(null, "Couldn't connect to I2P router.  Make sure I2P is running and restart MuWire",
-                "Can't connect to I2P router", JOptionPane.WARNING_MESSAGE)
+                    "Can't connect to I2P router", JOptionPane.WARNING_MESSAGE)
             System.exit(0)
         }
-        Runtime.getRuntime().addShutdownHook({
-            core.shutdown()
-        })
-        core.startServices()
-        application.context.put("muwire-settings", props)
-        application.context.put("core",core)
-        application.getPropertyChangeListeners("core").each {
-            it.propertyChange(new PropertyChangeEvent(this, "core", null, core))
-        }
-
-        core.eventBus.publish(new UILoadedEvent())
     }
     
     private String selectNickname() {
@@ -116,8 +118,8 @@ class Ready extends AbstractLifecycleHandler {
                         JOptionPane.WARNING_MESSAGE)
                 continue
             }
-            if (nickname.contains("@")) {
-                JOptionPane.showMessageDialog(null, "Nickname cannot contain @, choose another",
+            if (!DataUtil.isValidName(nickname)) {
+                JOptionPane.showMessageDialog(null, "Nickname cannot contain any of ${Constants.INVALID_NICKNAME_CHARS} choose another",
                         "Select another nickname", JOptionPane.WARNING_MESSAGE)
                 continue
             }

gui/griffon-app/models/com/muwire/gui/AdvancedSharingModel.groovy

@@ -1,32 +1,49 @@
 package com.muwire.gui
 
+import javax.annotation.Nonnull
 import javax.swing.tree.DefaultMutableTreeNode
 import javax.swing.tree.DefaultTreeModel
 import javax.swing.tree.MutableTreeNode
 
 import com.muwire.core.Core
 import com.muwire.core.files.FileTree
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectorySyncEvent
 
 import griffon.core.artifact.GriffonModel
+import griffon.inject.MVCMember
 import griffon.transform.Observable
 import griffon.metadata.ArtifactProviderFor
 
 @ArtifactProviderFor(GriffonModel)
 class AdvancedSharingModel {
+    
+    @MVCMember @Nonnull
+    AdvancedSharingView view
+    
     def watchedDirectories = []
     def treeRoot
     def negativeTree
     
     Core core
     
+    @Observable boolean syncActionEnabled
+    
     void mvcGroupInit(Map<String,String> args) {
-        watchedDirectories.addAll(core.muOptions.watchedDirectories)
+        watchedDirectories.addAll(core.watchedDirectoryManager.watchedDirs.values())
+        core.eventBus.register(WatchedDirectorySyncEvent.class, this)
+        core.eventBus.register(WatchedDirectoryConfigurationEvent.class, this)
         
         treeRoot = new DefaultMutableTreeNode()
         negativeTree = new DefaultTreeModel(treeRoot)
         copyTree(treeRoot, core.fileManager.negativeTree.root)
     }
     
+    void mvcGroupDestroy() {
+        core.eventBus.unregister(WatchedDirectorySyncEvent.class, this)
+        core.eventBus.unregister(WatchedDirectoryConfigurationEvent.class, this)
+    }
+    
     private void copyTree(DefaultMutableTreeNode jtreeNode, FileTree.TreeNode fileTreeNode) {
         jtreeNode.setUserObject(fileTreeNode.file?.getName())
         fileTreeNode.children.each { 
@@ -36,4 +53,16 @@ class AdvancedSharingModel {
         }
     }
     
+    void onWatchedDirectorySyncEvent(WatchedDirectorySyncEvent e) {
+        runInsideUIAsync {
+            view.watchedDirsTable.model.fireTableDataChanged()
+        }
+    }
+    
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        runInsideUIAsync {
+            view.watchedDirsTable.model.fireTableDataChanged()
+        }
+    }
+    
 }

gui/griffon-app/models/com/muwire/gui/FeedConfigurationModel.groovy

@@ -0,0 +1,26 @@
+package com.muwire.gui
+
+import com.muwire.core.Core
+import com.muwire.core.filefeeds.Feed
+
+import griffon.core.artifact.GriffonModel
+import griffon.transform.Observable
+import griffon.metadata.ArtifactProviderFor
+
+@ArtifactProviderFor(GriffonModel)
+class FeedConfigurationModel {
+    Core core
+    Feed feed
+    
+    @Observable boolean autoDownload
+    @Observable boolean sequential
+    @Observable int updateInterval
+    @Observable int itemsToKeep
+    
+    void mvcGroupInit(Map<String, String> args) {
+        autoDownload = feed.isAutoDownload()
+        sequential = feed.isSequential()
+        updateInterval = feed.getUpdateInterval() / 60000
+        itemsToKeep = feed.getItemsToKeep()
+    }
+}

gui/griffon-app/models/com/muwire/gui/FetchCertificatesModel.groovy

@@ -1,6 +1,9 @@
 package com.muwire.gui
 
+import com.muwire.core.InfoHash
+import com.muwire.core.Persona
 import com.muwire.core.filecert.CertificateFetchStatus
+import com.muwire.core.filefeeds.FeedItem
 import com.muwire.core.search.UIResultEvent
 
 import griffon.core.artifact.GriffonModel
@@ -9,7 +12,9 @@ import griffon.metadata.ArtifactProviderFor
 
 @ArtifactProviderFor(GriffonModel)
 class FetchCertificatesModel {
-    UIResultEvent result
+    Persona host
+    InfoHash infoHash
+    String name
     
     @Observable CertificateFetchStatus status
     @Observable int totalCertificates

gui/griffon-app/models/com/muwire/gui/MainFrameModel.groovy

@@ -28,6 +28,12 @@ import com.muwire.core.content.ContentControlEvent
 import com.muwire.core.download.DownloadStartedEvent
 import com.muwire.core.download.Downloader
 import com.muwire.core.filecert.CertificateCreatedEvent
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.FeedFetchEvent
+import com.muwire.core.filefeeds.FeedItemFetchedEvent
+import com.muwire.core.filefeeds.FeedLoadedEvent
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
 import com.muwire.core.files.AllFilesLoadedEvent
 import com.muwire.core.files.DirectoryUnsharedEvent
 import com.muwire.core.files.DirectoryWatchedEvent
@@ -61,6 +67,7 @@ import griffon.transform.FXObservable
 import griffon.transform.Observable
 import net.i2p.data.Base64
 import net.i2p.data.Destination
+import net.i2p.util.ConcurrentHashSet
 import griffon.metadata.ArtifactProviderFor
 
 @ArtifactProviderFor(GriffonModel)
@@ -89,6 +96,8 @@ class MainFrameModel {
     def trusted = []
     def distrusted = []
     def subscriptions = []
+    def feeds = []
+    def feedItems = []
     
     boolean sessionRestored
 
@@ -103,6 +112,14 @@ class MainFrameModel {
     @Observable boolean previewButtonEnabled
     @Observable String resumeButtonText
     @Observable boolean addCommentButtonEnabled
+    @Observable boolean publishButtonEnabled
+    @Observable String publishButtonText
+    @Observable boolean updateFileFeedButtonEnabled
+    @Observable boolean unsubscribeFileFeedButtonEnabled
+    @Observable boolean configureFileFeedButtonEnabled
+    @Observable boolean downloadFeedItemButtonEnabled
+    @Observable boolean viewFeedItemCommentButtonEnabled
+    @Observable boolean viewFeedItemCertificatesButtonEnabled
     @Observable boolean subscribeButtonEnabled
     @Observable boolean markNeutralFromTrustedButtonEnabled
     @Observable boolean markDistrustedButtonEnabled
@@ -118,6 +135,7 @@ class MainFrameModel {
     @Observable boolean downloadsPaneButtonEnabled
     @Observable boolean uploadsPaneButtonEnabled
     @Observable boolean monitorPaneButtonEnabled
+    @Observable boolean feedsPaneButtonEnabled
     @Observable boolean trustPaneButtonEnabled
     @Observable boolean chatPaneButtonEnabled
     
@@ -125,7 +143,7 @@ class MainFrameModel {
     
     @Observable Downloader downloader
 
-    private final Set<InfoHash> downloadInfoHashes = new HashSet<>()
+    private final Set<InfoHash> downloadInfoHashes = new ConcurrentHashSet<>()
 
     @Observable volatile Core core
 
@@ -215,6 +233,10 @@ class MainFrameModel {
             core.eventBus.register(TrustSubscriptionUpdatedEvent.class, this)
             core.eventBus.register(SearchEvent.class, this)
             core.eventBus.register(CertificateCreatedEvent.class, this)
+            core.eventBus.register(FeedLoadedEvent.class, this)
+            core.eventBus.register(FeedFetchEvent.class, this)
+            core.eventBus.register(FeedItemFetchedEvent.class, this)
+            core.eventBus.register(UIFeedConfigurationEvent.class, this)
 
             core.muOptions.watchedKeywords.each {
                 core.eventBus.publish(new ContentControlEvent(term : it, regex: false, add: true))
@@ -253,11 +275,13 @@ class MainFrameModel {
                 distrusted.addAll(core.trustService.bad.values())
 
                 resumeButtonText = "Retry"
+                publishButtonText = "Publish"
                 
                 searchesPaneButtonEnabled = false
                 downloadsPaneButtonEnabled = true
                 uploadsPaneButtonEnabled = true
                 monitorPaneButtonEnabled = true
+                feedsPaneButtonEnabled = true
                 trustPaneButtonEnabled = true
                 chatPaneButtonEnabled = true
                 
@@ -270,8 +294,6 @@ class MainFrameModel {
 
     void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
         runInsideUIAsync {
-            core.muOptions.watchedDirectories.each { core.eventBus.publish(new FileSharedEvent(file : new File(it))) }
-
             core.muOptions.trustSubscriptions.each {
                 core.eventBus.publish(new TrustSubscriptionEvent(persona : it, subscribe : true))
             }
@@ -391,7 +413,7 @@ class MainFrameModel {
                         break
                     if (parent.getChildCount() == 0) {
                         File file = parent.getUserObject().file
-                        if (core.muOptions.watchedDirectories.contains(file.toString()))
+                        if (core.watchedDirectoryManager.isWatched(file))
                             unshared.add(file)
                         dmtn = parent
                         continue
@@ -651,4 +673,41 @@ class MainFrameModel {
         int requests
         boolean finished
     }
+    
+    void onFeedLoadedEvent(FeedLoadedEvent e) {
+        runInsideUIAsync {
+            feeds << e.feed
+            view.refreshFeeds()
+        }
+    }
+    
+    void onFeedFetchEvent(FeedFetchEvent e) {
+        runInsideUIAsync {
+            view.refreshFeeds()
+        }
+    }
+    
+    void onUIFeedConfigurationEvent(UIFeedConfigurationEvent e) {
+        if (!e.newFeed)
+            return
+        runInsideUIAsync {
+            if (feeds.contains(e.feed))
+                return
+            feeds << e.feed
+            view.refreshFeeds()
+        }
+    }
+    
+    void onFeedItemFetchedEvent(FeedItemFetchedEvent e) {
+        Feed feed = core.feedManager.getFeed(e.item.getPublisher())
+        if (feed == null || !feed.isAutoDownload())
+            return
+        if (!canDownload(e.item.getInfoHash()))
+            return
+        if (core.fileManager.isShared(e.item.getInfoHash()))
+            return
+            
+        File target = new File(core.getMuOptions().getDownloadLocation(), e.item.getName())    
+        core.eventBus.publish(new UIDownloadFeedItemEvent(item : e.item, target : target, sequential : feed.isSequential()))
+    }
 }

gui/griffon-app/models/com/muwire/gui/OptionsModel.groovy

@@ -18,6 +18,7 @@ class OptionsModel {
     @Observable String incompleteLocation
     @Observable boolean searchComments
     @Observable boolean browseFiles
+    @Observable boolean allowTracking
     @Observable int speedSmoothSeconds
     @Observable int totalUploadSlots
     @Observable int uploadSlotsPerUser
@@ -50,6 +51,15 @@ class OptionsModel {
     @Observable String inBw
     @Observable String outBw
 
+    // feed options
+    @Observable boolean fileFeed
+    @Observable boolean advertiseFeed
+    @Observable boolean autoPublishSharedFiles
+    @Observable boolean defaultFeedAutoDownload
+    @Observable String defaultFeedItemsToKeep
+    @Observable boolean defaultFeedSequential
+    @Observable String defaultFeedUpdateInterval
+    
     // trust options
     @Observable boolean onlyTrusted
     @Observable boolean searchExtraHop
@@ -74,6 +84,7 @@ class OptionsModel {
         incompleteLocation = settings.incompleteLocation.getAbsolutePath()
         searchComments = settings.searchComments
         browseFiles = settings.browseFiles
+        allowTracking = settings.allowTracking
         speedSmoothSeconds = settings.speedSmoothSeconds
         totalUploadSlots = settings.totalUploadSlots
         uploadSlotsPerUser = settings.uploadSlotsPerUser
@@ -105,6 +116,14 @@ class OptionsModel {
             inBw = String.valueOf(settings.inBw)
             outBw = String.valueOf(settings.outBw)
         }
+        
+        fileFeed = settings.fileFeed
+        advertiseFeed = settings.advertiseFeed
+        autoPublishSharedFiles = settings.autoPublishSharedFiles
+        defaultFeedAutoDownload = settings.defaultFeedAutoDownload
+        defaultFeedItemsToKeep = String.valueOf(settings.defaultFeedItemsToKeep)
+        defaultFeedSequential = settings.defaultFeedSequential
+        defaultFeedUpdateInterval = String.valueOf(settings.defaultFeedUpdateInterval)
 
         onlyTrusted = !settings.allowUntrusted()
         searchExtraHop = settings.searchExtraHop

gui/griffon-app/models/com/muwire/gui/SearchTabModel.groovy

@@ -25,6 +25,7 @@ class SearchTabModel {
     @Observable boolean viewCommentActionEnabled
     @Observable boolean viewCertificatesActionEnabled
     @Observable boolean chatActionEnabled
+    @Observable boolean subscribeActionEnabled
     @Observable boolean groupedByFile
 
     Core core

gui/griffon-app/models/com/muwire/gui/SignModel.groovy

@@ -0,0 +1,9 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonModel
+import griffon.transform.Observable
+import griffon.metadata.ArtifactProviderFor
+
+@ArtifactProviderFor(GriffonModel)
+class SignModel {
+}

gui/griffon-app/models/com/muwire/gui/WatchedDirectoryModel.groovy

@@ -0,0 +1,22 @@
+package com.muwire.gui
+
+import com.muwire.core.Core
+import com.muwire.core.files.directories.WatchedDirectory
+
+import griffon.core.artifact.GriffonModel
+import griffon.transform.Observable
+import griffon.metadata.ArtifactProviderFor
+
+@ArtifactProviderFor(GriffonModel)
+class WatchedDirectoryModel {
+    Core core
+    WatchedDirectory directory
+    
+    @Observable boolean autoWatch
+    @Observable int syncInterval
+    
+    void mvcGroupInit(Map<String,String> args) {
+        autoWatch = directory.autoWatch
+        syncInterval = directory.syncInterval
+    }
+}

gui/griffon-app/views/com/muwire/gui/AdvancedSharingView.groovy

@@ -3,13 +3,23 @@ package com.muwire.gui
 import griffon.core.artifact.GriffonView
 import griffon.inject.MVCMember
 import griffon.metadata.ArtifactProviderFor
+import net.i2p.data.DataHelper
 
 import javax.swing.JDialog
+import javax.swing.JLabel
+import javax.swing.JMenuItem
+import javax.swing.JPopupMenu
 import javax.swing.JTabbedPane
 import javax.swing.JTree
+import javax.swing.ListSelectionModel
 import javax.swing.SwingConstants
+import javax.swing.table.DefaultTableCellRenderer
+
+import com.muwire.core.files.directories.WatchedDirectory
 
 import java.awt.BorderLayout
+import java.awt.event.MouseAdapter
+import java.awt.event.MouseEvent
 import java.awt.event.WindowAdapter
 import java.awt.event.WindowEvent
 
@@ -21,6 +31,8 @@ class AdvancedSharingView {
     FactoryBuilderSupport builder
     @MVCMember @Nonnull
     AdvancedSharingModel model
+    @MVCMember @Nonnull
+    AdvancedSharingController controller
 
     def mainFrame
     def dialog
@@ -28,6 +40,7 @@ class AdvancedSharingView {
     def negativeTreePanel
     
     def watchedDirsTable
+    def watchedDirsTableSortEvent
     
     void initUI() {
         mainFrame = application.windowManager.findWindow("main-frame")
@@ -43,10 +56,17 @@ class AdvancedSharingView {
             scrollPane( constraints : BorderLayout.CENTER ) {
                 watchedDirsTable = table(autoCreateRowSorter : true, rowHeight : rowHeight) {
                     tableModel(list : model.watchedDirectories) {
-                        closureColumn(header : "Directory", type : String, read : {it})
+                        closureColumn(header : "Directory", preferredWidth: 350, type : String, read : {it.directory.toString()})
+                        closureColumn(header : "Auto", preferredWidth: 100, type : Boolean, read : {it.autoWatch})
+                        closureColumn(header : "Interval", preferredWidth : 100, type : Integer, read : {it.syncInterval})
+                        closureColumn(header : "Last Sync", preferredWidth: 250, type : Long, read : {it.lastSync})
                     }
                 }
             }
+            panel (constraints : BorderLayout.SOUTH) {
+                button(text : "Configure", configureAction)
+                button(text : "Sync", enabled : bind{model.syncActionEnabled}, syncAction)
+            }
         }
         
         negativeTreePanel = builder.panel {
@@ -59,6 +79,54 @@ class AdvancedSharingView {
                 tree(rootVisible : false, rowHeight : rowHeight,jtree)
             }
         }
+        
+        def centerRenderer = new DefaultTableCellRenderer()
+        centerRenderer.setHorizontalAlignment(JLabel.CENTER)
+        watchedDirsTable.setDefaultRenderer(Long.class, new DateRenderer())
+        watchedDirsTable.setDefaultRenderer(Integer.class, centerRenderer)
+        
+        watchedDirsTable.rowSorter.addRowSorterListener({evt -> watchedDirsTableSortEvent = evt})
+        def selectionModel = watchedDirsTable.getSelectionModel()
+        selectionModel.setSelectionMode(ListSelectionModel.SINGLE_SELECTION)
+        selectionModel.addListSelectionListener({
+            def directory = selectedWatchedDirectory()
+            model.syncActionEnabled = !(directory == null || directory.autoWatch)
+        })
+        
+        watchedDirsTable.addMouseListener(new MouseAdapter() {
+            public void mouseReleased(MouseEvent e) {
+                if (e.isPopupTrigger())
+                    showMenu(e)
+            }
+            public void mousePressed(MouseEvent e) {
+                if (e.isPopupTrigger())
+                    showMenu(e)
+            }
+        })
+    }
+    
+    private void showMenu(MouseEvent e) {
+        JPopupMenu menu = new JPopupMenu()
+        JMenuItem configure = new JMenuItem("Configure")
+        configure.addActionListener({controller.configure()})
+        menu.add(configure)
+        
+        if (model.syncActionEnabled) {
+            JMenuItem sync = new JMenuItem("Sync")
+            sync.addActionListener({controller.sync()})
+            menu.add(sync)
+        }
+        
+        menu.show(e.getComponent(), e.getX(), e.getY())
+    }
+    
+    WatchedDirectory selectedWatchedDirectory() {
+        int row = watchedDirsTable.getSelectedRow()
+        if (row < 0)
+            return null
+        if (watchedDirsTableSortEvent != null)
+            row = watchedDirsTable.rowSorter.convertRowIndexToModel(row)
+        model.watchedDirectories[row]
     }
     
     void mvcGroupInit(Map<String,String> args) {

gui/griffon-app/views/com/muwire/gui/FeedConfigurationView.groovy

@@ -0,0 +1,73 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonView
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+
+import javax.swing.JDialog
+import javax.swing.SwingConstants
+
+import java.awt.BorderLayout
+import java.awt.GridBagConstraints
+import java.awt.event.WindowAdapter
+import java.awt.event.WindowEvent
+
+import javax.annotation.Nonnull
+
+@ArtifactProviderFor(GriffonView)
+class FeedConfigurationView {
+    @MVCMember @Nonnull
+    FactoryBuilderSupport builder
+    @MVCMember @Nonnull
+    FeedConfigurationModel model
+
+    def dialog
+    def p
+    def mainFrame
+    
+    def autoDownloadCheckbox
+    def sequentialCheckbox
+    def itemsToKeepField
+    def updateIntervalField
+    
+    void initUI() {
+        mainFrame = application.windowManager.findWindow("main-frame")
+        dialog = new JDialog(mainFrame, "Feed Configuration", true)
+        dialog.setResizable(false)
+        
+        p = builder.panel {
+            borderLayout()
+            panel (constraints : BorderLayout.NORTH) {
+                label("Configuration for feed " + model.feed.getPublisher().getHumanReadableName())
+            }
+            panel (constraints : BorderLayout.CENTER) {
+                gridBagLayout()
+                label(text : "Automatically download files from feed", constraints : gbc(gridx: 0, gridy : 0, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                autoDownloadCheckbox = checkBox(selected : bind {model.autoDownload}, constraints : gbc(gridx: 1, gridy : 0, anchor : GridBagConstraints.LINE_END))
+                label(text : "Download files from feed sequentially", constraints : gbc(gridx: 0, gridy : 1, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                sequentialCheckbox = checkBox(selected : bind {model.sequential}, constraints : gbc(gridx: 1, gridy : 1, anchor : GridBagConstraints.LINE_END))
+                label(text : "Feed items to store on disk (-1 means unlimited)", constraints : gbc(gridx: 0, gridy : 2, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                itemsToKeepField = textField(text : bind {model.itemsToKeep}, constraints:gbc(gridx :1, gridy:2, anchor : GridBagConstraints.LINE_END))
+                label(text : "Feed refresh frequency in minutes", constraints : gbc(gridx: 0, gridy : 3, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                updateIntervalField = textField(text : bind {model.updateInterval}, constraints:gbc(gridx :1, gridy:3, anchor : GridBagConstraints.LINE_END))
+            }
+            panel (constraints : BorderLayout.SOUTH) {
+                button(text : "Save", saveAction)
+                button(text : "Cancel", cancelAction)
+            } 
+        }
+    }
+    
+    void mvcGroupInit(Map<String,String> args) {
+        dialog.getContentPane().add(p)
+        dialog.pack()
+        dialog.setLocationRelativeTo(mainFrame)
+        dialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
+        dialog.addWindowListener(new WindowAdapter() {
+            public void windowClosed(WindowEvent e) {
+                mvcGroup.destroy()
+            }
+        })
+        dialog.show()
+    }
+}

gui/griffon-app/views/com/muwire/gui/FetchCertificatesView.groovy

@@ -38,7 +38,7 @@ class FetchCertificatesView {
     void initUI() {
         int rowHeight = application.context.get("row-height")
         mainFrame = application.windowManager.findWindow("main-frame")
-        dialog = new JDialog(mainFrame, model.result.name, true)
+        dialog = new JDialog(mainFrame, model.name, true)
         dialog.setResizable(true)
         
         p = builder.panel {

gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy

@@ -39,6 +39,9 @@ import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
 import com.muwire.core.SharedFile
 import com.muwire.core.download.Downloader
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.FeedFetchStatus
+import com.muwire.core.filefeeds.FeedItem
 import com.muwire.core.files.FileSharedEvent
 import com.muwire.core.trust.RemoteTrustList
 import java.awt.BorderLayout
@@ -76,6 +79,8 @@ class MainFrameView {
     def lastSharedSortEvent
     def trustTablesSortEvents = [:]
     def expansionListener = new TreeExpansions()
+    def lastFeedsSortEvent
+    def lastFeedItemsSortEvent
     
 
     UISettings settings
@@ -139,6 +144,11 @@ class MainFrameView {
                                 mvcGroup.createMVCGroup("chat-monitor","chat-monitor",env)
                             }
                         })
+                        menuItem("Sign Tool", actionPerformed : {
+                            def env = [:]
+                            env['core'] = model.core
+                            mvcGroup.createMVCGroup("sign",env)
+                        })
                     }
                 }
                 borderLayout()
@@ -151,6 +161,7 @@ class MainFrameView {
                         button(text: "Uploads", enabled : bind{model.uploadsPaneButtonEnabled}, actionPerformed : showUploadsWindow)
                         if (settings.showMonitor)
                             button(text: "Monitor", enabled: bind{model.monitorPaneButtonEnabled},actionPerformed : showMonitorWindow)
+                        button(text: "Feeds", enabled: bind {model.feedsPaneButtonEnabled}, actionPerformed : showFeedsWindow)
                         button(text: "Trust", enabled:bind{model.trustPaneButtonEnabled},actionPerformed : showTrustWindow)
                         button(text: "Chat", enabled : bind{model.chatPaneButtonEnabled}, actionPerformed : showChatWindow)
                     }
@@ -292,6 +303,7 @@ class MainFrameView {
                                                         Core core = application.context.get("core")
                                                         core.certificateManager.hasLocalCertificate(new InfoHash(it.getRoot()))
                                                     })
+                                                    closureColumn(header : "Published", preferredWidth : 50, type : Boolean, read : {row -> row.isPublished()})
                                                     closureColumn(header : "Search Hits", preferredWidth: 50, type : Integer, read : {it.getHits()})
                                                     closureColumn(header : "Downloaders", preferredWidth: 50, type : Integer, read : {it.getDownloaders().size()})
                                                 }
@@ -316,9 +328,11 @@ class MainFrameView {
                                     radioButton(text : "Table", selected : false, buttonGroup : sharedViewType, actionPerformed : showSharedFilesTable)
                                 }
                                 panel {
-                                    button(text : "Share", actionPerformed : shareFiles)
-                                    button(text : "Add Comment", enabled : bind {model.addCommentButtonEnabled}, addCommentAction)
-                                    button(text : "Certify", enabled : bind {model.addCommentButtonEnabled}, issueCertificateAction)
+                                    gridBagLayout()
+                                    button(text : "Share", constraints : gbc(gridx: 0), actionPerformed : shareFiles)
+                                    button(text : "Add Comment", enabled : bind {model.addCommentButtonEnabled}, constraints : gbc(gridx: 1), addCommentAction)
+                                    button(text : "Certify", enabled : bind {model.addCommentButtonEnabled}, constraints : gbc(gridx: 2), issueCertificateAction)
+                                    button(text : bind {model.publishButtonText}, enabled : bind {model.publishButtonEnabled}, constraints : gbc(gridx:3), publishAction)
                                 }
                                 panel {
                                     panel {
@@ -425,6 +439,56 @@ class MainFrameView {
                             }
                         }
                     }
+                    panel(constraints : "feeds window") {
+                        gridLayout(rows : 2, cols : 1)
+                        panel {
+                            borderLayout()
+                            panel (constraints : BorderLayout.NORTH) {
+                                label(text: "Subscriptions")
+                            }
+                            scrollPane(constraints : BorderLayout.CENTER) {
+                                table(id : "feeds-table", autoCreateRowSorter : true, rowHeight : rowHeight) {
+                                    tableModel(list : model.feeds) {
+                                        closureColumn(header : "Publisher", preferredWidth: 350, type : String, read : {it.getPublisher().getHumanReadableName()})
+                                        closureColumn(header : "Files", preferredWidth: 10, type : Integer, read : {model.core.feedManager.getFeedItems(it.getPublisher()).size()})
+                                        closureColumn(header : "Last Updated", type : Long, read : {it.getLastUpdated()})
+                                        closureColumn(header : "Status", preferredWidth: 10, type : String, read : {it.getStatus().toString()})
+                                    }
+                                }
+                            }
+                            panel (constraints : BorderLayout.SOUTH) {
+                                button(text : "Update", enabled : bind {model.updateFileFeedButtonEnabled}, updateFileFeedAction)
+                                button(text : "Unsubscribe", enabled : bind {model.unsubscribeFileFeedButtonEnabled}, unsubscribeFileFeedAction)
+                                button(text : "Configure", enabled : bind {model.configureFileFeedButtonEnabled}, configureFileFeedAction)
+                            }
+                        }
+                        panel {
+                            borderLayout()
+                            panel (constraints : BorderLayout.NORTH) {
+                                label(text : "Published Files")
+                            }
+                            scrollPane(constraints : BorderLayout.CENTER) {
+                                table(id : "feed-items-table", autoCreateRowSorter : true, rowHeight : rowHeight) {
+                                    tableModel(list : model.feedItems) {
+                                        closureColumn(header : "Name", preferredWidth: 350, type : String, read : {it.getName()})
+                                        closureColumn(header : "Size", preferredWidth: 10, type : Long, read : {it.getSize()})
+                                        closureColumn(header : "Comment", preferredWidth: 10, type : Boolean, read : {it.getComment() != null})
+                                        closureColumn(header : "Certificates", preferredWidth: 10, type : Integer, read : {it.getCertificates()})
+                                        closureColumn(header : "Downloaded", preferredWidth: 10, type : Boolean, read : {
+                                            InfoHash ih = it.getInfoHash()
+                                            model.core.fileManager.isShared(ih)
+                                        })
+                                        closureColumn(header: "Date", type : Long, read : {it.getTimestamp()})
+                                    }
+                                }
+                            }
+                            panel(constraints : BorderLayout.SOUTH) {
+                                button(text : "Download", enabled : bind {model.downloadFeedItemButtonEnabled}, downloadFeedItemAction)
+                                button(text : "View Comment", enabled : bind {model.viewFeedItemCommentButtonEnabled}, viewFeedItemCommentAction)
+                                button(text : "View Certificates", enabled : bind {model.viewFeedItemCertificatesButtonEnabled}, viewFeedItemCertificatesAction )
+                            }
+                        }
+                    }
                     panel(constraints : "trust window") {
                         gridLayout(rows : 2, cols : 1)
                         panel {
@@ -682,14 +746,30 @@ class MainFrameView {
             if (selectedFiles == null || selectedFiles.isEmpty())
                 return
             model.addCommentButtonEnabled = true
+            model.publishButtonEnabled = true
+            boolean unpublish = true
+            selectedFiles.each { 
+                unpublish &= it.isPublished()
+            }
+            model.publishButtonText = unpublish ? "Unpublish" : "Publish"
         })
+        
         def sharedFilesTree = builder.getVariable("shared-files-tree")
         sharedFilesTree.addMouseListener(sharedFilesMouseListener)
 
         sharedFilesTree.addTreeSelectionListener({
             def selectedNode = sharedFilesTree.getLastSelectedPathComponent()
             model.addCommentButtonEnabled = selectedNode != null
-
+            model.publishButtonEnabled = selectedNode != null
+            
+            def selectedFiles = selectedSharedFiles()
+            if (selectedFiles == null || selectedFiles.isEmpty())
+                return
+            boolean unpublish = true
+            selectedFiles.each {
+                unpublish &= it.isPublished()
+            }
+            model.publishButtonText = unpublish ? "Unpublish" : "Publish"
         })
         
         sharedFilesTree.addTreeExpansionListener(expansionListener)
@@ -745,6 +825,98 @@ class MainFrameView {
                     }
                 })
 
+        // feeds table
+        def feedsTable = builder.getVariable("feeds-table")
+        feedsTable.rowSorter.addRowSorterListener({evt -> lastFeedsSortEvent = evt})
+        feedsTable.rowSorter.setSortsOnUpdates(true)
+        feedsTable.setDefaultRenderer(Integer.class, centerRenderer)
+        feedsTable.setDefaultRenderer(Long.class, new DateRenderer())
+        selectionModel = feedsTable.getSelectionModel()
+        selectionModel.setSelectionMode(ListSelectionModel.SINGLE_SELECTION)
+        selectionModel.addListSelectionListener({
+            Feed selectedFeed = selectedFeed()
+            if (selectedFeed == null) {
+                model.updateFileFeedButtonEnabled = false
+                model.unsubscribeFileFeedButtonEnabled = false
+                model.configureFileFeedButtonEnabled = false
+                return
+            }
+
+            model.unsubscribeFileFeedButtonEnabled = true
+            model.configureFileFeedButtonEnabled = true
+            model.updateFileFeedButtonEnabled = !selectedFeed.getStatus().isActive()
+            
+            def items = model.core.feedManager.getFeedItems(selectedFeed.getPublisher())
+            model.feedItems.clear()
+            model.feedItems.addAll(items)
+            
+            def feedItemsTable = builder.getVariable("feed-items-table")
+            int selectedItemRow = feedItemsTable.getSelectedRow()
+            feedItemsTable.model.fireTableDataChanged()
+            if (selectedItemRow >= 0 && selectedItemRow < items.size())
+                feedItemsTable.selectionModel.setSelectionInterval(selectedItemRow, selectedItemRow)
+        })
+        feedsTable.addMouseListener(new MouseAdapter() {
+            @Override
+            public void mouseClicked(MouseEvent e) {
+                if(e.isPopupTrigger() || e.getButton() == MouseEvent.BUTTON3)
+                    showFeedsPopupMenu(e)
+            }
+            @Override
+            public void mouseReleased(MouseEvent e) {
+                if(e.isPopupTrigger() || e.getButton() == MouseEvent.BUTTON3)
+                    showFeedsPopupMenu(e)
+            }
+        })
+        
+        
+        // feed items table
+        def feedItemsTable = builder.getVariable("feed-items-table")
+        feedItemsTable.rowSorter.addRowSorterListener({evt -> lastFeedItemsSortEvent = evt})
+        feedItemsTable.rowSorter.setSortsOnUpdates(true)
+        feedItemsTable.setDefaultRenderer(Integer.class, centerRenderer)
+        feedItemsTable.columnModel.getColumn(1).setCellRenderer(new SizeRenderer())
+        feedItemsTable.columnModel.getColumn(5).setCellRenderer(new DateRenderer())
+        
+        selectionModel = feedItemsTable.getSelectionModel()
+        selectionModel.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION)
+        selectionModel.addListSelectionListener({
+            List<FeedItem> selectedItems = selectedFeedItems()
+            if (selectedItems == null || selectedItems.isEmpty()) {
+                model.downloadFeedItemButtonEnabled = false
+                model.viewFeedItemCommentButtonEnabled = false
+                model.viewFeedItemCertificatesButtonEnabled = false
+                return
+            }
+            model.downloadFeedItemButtonEnabled = true
+            model.viewFeedItemCommentButtonEnabled = false
+            model.viewFeedItemCertificatesButtonEnabled = false
+            if (selectedItems.size() == 1) {
+                FeedItem item = selectedItems.get(0)
+                model.viewFeedItemCommentButtonEnabled = item.getComment() != null
+                model.viewFeedItemCertificatesButtonEnabled = item.getCertificates() > 0
+            }
+        })
+        feedItemsTable.addMouseListener(new MouseAdapter() {
+            @Override
+            public void mouseClicked(MouseEvent e) {
+                List<FeedItem> selectedItems = selectedFeedItems()
+                if (e.isPopupTrigger() || e.getButton() == MouseEvent.BUTTON3)
+                    showFeedItemsPopupMenu(e)
+                else if (e.getButton() == MouseEvent.BUTTON1 && e.getClickCount() == 2 &&
+                    selectedItems != null && selectedItems.size() == 1 &&
+                    model.canDownload(selectedItems.get(0).getInfoHash())) {
+                    mvcGroup.controller.downloadFeedItem()
+                }
+            }
+            
+            @Override
+            public void mouseReleased(MouseEvent e) {
+                if (e.isPopupTrigger() || e.getButton() == MouseEvent.BUTTON3)
+                    showFeedItemsPopupMenu(e)
+            }
+        })
+        
         // subscription table
         def subscriptionTable = builder.getVariable("subscription-table")
         subscriptionTable.setDefaultRenderer(Integer.class, centerRenderer)
@@ -1007,6 +1179,52 @@ class MainFrameView {
         showPopupMenu(menu, e)
     }
     
+    void showFeedsPopupMenu(MouseEvent e) {
+        Feed feed = selectedFeed()
+        if (feed == null)
+            return
+        JPopupMenu menu = new JPopupMenu()
+        if (model.updateFileFeedButtonEnabled) {
+            JMenuItem update = new JMenuItem("Update")
+            update.addActionListener({mvcGroup.controller.updateFileFeed()})
+            menu.add(update)
+        }
+        
+        JMenuItem unsubscribe = new JMenuItem("Unsubscribe")
+        unsubscribe.addActionListener({mvcGroup.controller.unsubscribeFileFeed()})
+        menu.add(unsubscribe)
+        
+        JMenuItem configure = new JMenuItem("Configure")
+        configure.addActionListener({mvcGroup.controller.configureFileFeed()})
+        menu.add(configure)
+        
+        showPopupMenu(menu,e)
+    }
+    
+    void showFeedItemsPopupMenu(MouseEvent e) {
+        List<FeedItem> items = selectedFeedItems()
+        if (items == null || items.isEmpty())
+            return
+        // TODO: finish
+        JPopupMenu menu = new JPopupMenu()
+        if (model.downloadFeedItemButtonEnabled) {
+            JMenuItem download = new JMenuItem("Download")
+            download.addActionListener({mvcGroup.controller.downloadFeedItem()})
+            menu.add(download)
+        }
+        if (model.viewFeedItemCommentButtonEnabled) {
+            JMenuItem viewComment = new JMenuItem("View Comment")
+            viewComment.addActionListener({mvcGroup.controller.viewFeedItemComment()})
+            menu.add(viewComment)
+        }
+        if (model.viewFeedItemCertificatesButtonEnabled) {
+            JMenuItem viewCertificates = new JMenuItem("View Certificates")
+            viewCertificates.addActionListener({mvcGroup.controller.viewFeedItemCertificates()})
+            menu.add(viewCertificates)
+        }
+        showPopupMenu(menu, e)
+    }
+    
     def selectedUploader() {
         def uploadsTable = builder.getVariable("uploads-table")
         int selectedRow = uploadsTable.getSelectedRow()
@@ -1057,6 +1275,7 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = true
         model.uploadsPaneButtonEnabled = true
         model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = true
         model.trustPaneButtonEnabled = true
         model.chatPaneButtonEnabled = true
         chatNotificator.mainWindowDeactivated()
@@ -1069,6 +1288,7 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = false
         model.uploadsPaneButtonEnabled = true
         model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = true
         model.trustPaneButtonEnabled = true
         model.chatPaneButtonEnabled = true
         chatNotificator.mainWindowDeactivated()
@@ -1081,6 +1301,7 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = true
         model.uploadsPaneButtonEnabled = false
         model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = true
         model.trustPaneButtonEnabled = true
         model.chatPaneButtonEnabled = true
         chatNotificator.mainWindowDeactivated()
@@ -1093,6 +1314,20 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = true
         model.uploadsPaneButtonEnabled = true
         model.monitorPaneButtonEnabled = false
+        model.feedsPaneButtonEnabled = true
+        model.trustPaneButtonEnabled = true
+        model.chatPaneButtonEnabled = true
+        chatNotificator.mainWindowDeactivated()
+    }
+    
+    def showFeedsWindow = {
+        def cardsPanel = builder.getVariable("cards-panel")
+        cardsPanel.getLayout().show(cardsPanel,"feeds window")
+        model.searchesPaneButtonEnabled = true
+        model.downloadsPaneButtonEnabled = true
+        model.uploadsPaneButtonEnabled = true
+        model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = false
         model.trustPaneButtonEnabled = true
         model.chatPaneButtonEnabled = true
         chatNotificator.mainWindowDeactivated()
@@ -1105,6 +1340,7 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = true
         model.uploadsPaneButtonEnabled = true
         model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = true
         model.trustPaneButtonEnabled = false
         model.chatPaneButtonEnabled = true
         chatNotificator.mainWindowDeactivated()
@@ -1117,6 +1353,7 @@ class MainFrameView {
         model.downloadsPaneButtonEnabled = true
         model.uploadsPaneButtonEnabled = true
         model.monitorPaneButtonEnabled = true
+        model.feedsPaneButtonEnabled = true
         model.trustPaneButtonEnabled = true
         model.chatPaneButtonEnabled = false
         chatNotificator.mainWindowActivated()
@@ -1173,6 +1410,43 @@ class MainFrameView {
         builder.getVariable("shared-files-table").model.fireTableDataChanged()
     }
     
+    public void refreshFeeds() {
+        JTable feedsTable = builder.getVariable("feeds-table")
+        int selectedFeed = feedsTable.getSelectedRow()
+        feedsTable.model.fireTableDataChanged()
+        if (selectedFeed >= 0)
+            feedsTable.selectionModel.setSelectionInterval(selectedFeed, selectedFeed)
+        
+        JTable feedItemsTable = builder.getVariable("feed-items-table")
+        feedItemsTable.model.fireTableDataChanged()
+    }
+    
+    Feed selectedFeed() {
+        JTable feedsTable = builder.getVariable("feeds-table")
+        int row = feedsTable.getSelectedRow()
+        if (row < 0)
+            return null
+        if (lastFeedsSortEvent != null)
+            row = feedsTable.rowSorter.convertRowIndexToModel(row)
+        model.feeds[row]
+    }
+    
+    List<FeedItem> selectedFeedItems() {
+        JTable feedItemsTable = builder.getVariable("feed-items-table")
+        int [] selectedRows = feedItemsTable.getSelectedRows()
+        if (selectedRows.length == 0)
+            return null
+        List<FeedItem> rv = new ArrayList<>()
+        if (lastFeedItemsSortEvent != null) {
+            for (int i = 0; i < selectedRows.length; i++) {
+                selectedRows[i] = feedItemsTable.rowSorter.convertRowIndexToModel(selectedRows[i])
+            }
+        }
+        for (int selectedRow : selectedRows)
+            rv.add(model.feedItems[selectedRow])
+        rv
+    }
+    
     private void closeApplication() {
         Core core = application.getContext().get("core")
         

gui/griffon-app/views/com/muwire/gui/OptionsView.groovy

@@ -32,6 +32,7 @@ class OptionsView {
     def i
     def u
     def bandwidth
+    def feed
     def trust
     def chat
 
@@ -42,6 +43,7 @@ class OptionsView {
     def shareHiddenCheckbox
     def searchCommentsCheckbox
     def browseFilesCheckbox
+    def allowTrackingCheckbox
     def speedSmoothSecondsField
     def totalUploadSlotsField
     def uploadSlotsPerUserField
@@ -66,6 +68,14 @@ class OptionsView {
 
     def inBwField
     def outBwField
+    
+    def fileFeedCheckbox
+    def advertiseFeedCheckbox
+    def autoPublishSharedFilesCheckbox
+    def defaultFeedAutoDownloadCheckbox
+    def defaultFeedItemsToKeepField
+    def defaultFeedSequentialCheckbox
+    def defaultFeedUpdateIntervalField
 
     def allowUntrustedCheckbox
     def searchExtraHopCheckbox
@@ -98,6 +108,10 @@ class OptionsView {
                     fill : GridBagConstraints.HORIZONTAL, weightx: 100))
                 browseFilesCheckbox = checkBox(selected : bind {model.browseFiles}, constraints : gbc(gridx : 1, gridy : 1,
                 anchor : GridBagConstraints.LINE_END, fill : GridBagConstraints.HORIZONTAL, weightx: 0))
+                label(text : "Allow tracking", constraints : gbc(gridx: 0, gridy: 2, anchor: GridBagConstraints.LINE_START,
+                    fill : GridBagConstraints.HORIZONTAL, weightx: 100))
+                allowTrackingCheckbox = checkBox(selected : bind {model.allowTracking}, constraints : gbc(gridx: 1, gridy : 2,
+                    anchor : GridBagConstraints.LINE_END, fill : GridBagConstraints.HORIZONTAL, weightx : 0))
             }
             
             panel (border : titledBorder(title : "Download Settings", border : etchedBorder(), titlePosition : TitledBorder.TOP,
@@ -257,6 +271,32 @@ class OptionsView {
             }
             panel(constraints : gbc(gridx: 0, gridy: 1, weighty: 100))
         }
+        feed = builder.panel {
+            gridBagLayout()
+            panel (border : titledBorder(title : "General Feed Settings", border : etchedBorder(), titlePosition : TitledBorder.TOP),
+                constraints : gbc(gridx : 0, gridy : 0, fill : GridBagConstraints.HORIZONTAL, weightx: 100)) {
+                gridBagLayout()
+                label(text : "Enable file feed", constraints : gbc(gridx: 0, gridy : 0, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                fileFeedCheckbox = checkBox(selected : bind {model.fileFeed}, constraints : gbc(gridx: 1, gridy : 0, anchor : GridBagConstraints.LINE_END))
+                label(text : "Advertise feed in search results", constraints : gbc(gridx: 0, gridy : 1, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                advertiseFeedCheckbox = checkBox(selected : bind {model.advertiseFeed}, constraints : gbc(gridx: 1, gridy : 1, anchor : GridBagConstraints.LINE_END))
+                label(text : "Automatically publish shared files", constraints : gbc(gridx: 0, gridy : 2, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                autoPublishSharedFilesCheckbox = checkBox(selected : bind {model.autoPublishSharedFiles}, constraints : gbc(gridx: 1, gridy : 2, anchor : GridBagConstraints.LINE_END))
+            }
+            panel (border : titledBorder(title : "Default Settings For New Feeds", border : etchedBorder(), titlePosition : TitledBorder.TOP),
+                constraints : gbc(gridx : 0, gridy : 1, fill : GridBagConstraints.HORIZONTAL, weightx: 100)) {
+                gridBagLayout()
+                label(text : "Automatically download files from feed", constraints : gbc(gridx: 0, gridy : 0, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                defaultFeedAutoDownloadCheckbox = checkBox(selected : bind {model.defaultFeedAutoDownload}, constraints : gbc(gridx: 1, gridy : 0, anchor : GridBagConstraints.LINE_END))
+                label(text : "Download files from feed sequentially", constraints : gbc(gridx: 0, gridy : 1, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                defaultFeedSequentialCheckbox = checkBox(selected : bind {model.defaultFeedSequential}, constraints : gbc(gridx: 1, gridy : 1, anchor : GridBagConstraints.LINE_END))
+                label(text : "Feed items to store on disk (-1 means unlimited)", constraints : gbc(gridx: 0, gridy : 2, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                defaultFeedItemsToKeepField = textField(text : bind {model.defaultFeedItemsToKeep}, constraints:gbc(gridx :1, gridy:2, anchor : GridBagConstraints.LINE_END))
+                label(text : "Feed refresh frequency in minutes", constraints : gbc(gridx: 0, gridy : 3, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                defaultFeedUpdateIntervalField = textField(text : bind {model.defaultFeedUpdateInterval}, constraints:gbc(gridx :1, gridy:3, anchor : GridBagConstraints.LINE_END))
+            }
+            panel(constraints : gbc(gridx: 0, gridy : 2, weighty: 100))
+        }
         trust = builder.panel {
             gridBagLayout()
             panel (border : titledBorder(title : "Trust Settings", border : etchedBorder(), titlePosition : TitledBorder.TOP),
@@ -311,6 +351,7 @@ class OptionsView {
         if (core.router != null) {
             tabbedPane.addTab("Bandwidth", bandwidth)
         }
+        tabbedPane.addTab("Feed", feed)
         tabbedPane.addTab("Trust", trust)
         tabbedPane.addTab("Chat", chat)
 

gui/griffon-app/views/com/muwire/gui/SearchTabView.groovy

@@ -74,6 +74,7 @@ class SearchTabView {
                                             closureColumn(header : "Sender", preferredWidth : 500, type: String, read : {row -> row.getHumanReadableName()})
                                             closureColumn(header : "Results", preferredWidth : 20, type: Integer, read : {row -> model.sendersBucket[row].size()})
                                             closureColumn(header : "Browse", preferredWidth : 20, type: Boolean, read : {row -> model.sendersBucket[row].first().browse})
+                                            closureColumn(header : "Feed", preferredWidth : 20, type : Boolean, read : {row -> model.sendersBucket[row].first().feed})
                                             closureColumn(header : "Chat", preferredWidth : 20, type : Boolean, read : {row -> model.sendersBucket[row].first().chat})
                                             closureColumn(header : "Trust", preferredWidth : 50, type: String, read : { row ->
                                                 model.core.trustService.getLevel(row.destination).toString()
@@ -85,6 +86,7 @@ class SearchTabView {
                                     gridLayout(rows: 1, cols : 2)
                                     panel (border : etchedBorder()){
                                         button(text : "Browse Host", enabled : bind {model.browseActionEnabled}, browseAction)
+                                        button(text : "Subscribe", enabled : bind {model.subscribeActionEnabled}, subscribeAction)
                                         button(text : "Chat", enabled : bind{model.chatActionEnabled}, chatAction)
                                     }
                                     panel (border : etchedBorder()){
@@ -156,6 +158,14 @@ class SearchTabView {
                                                 }
                                                 count
                                             })
+                                            closureColumn(header : "Feeds", preferredWidth : 20, type : Integer, read : {
+                                                int count = 0
+                                                model.hashBucket[it].each { 
+                                                    if (it.feed)
+                                                        count++
+                                                }
+                                                count
+                                            })
                                             closureColumn(header : "Chat Hosts", preferredWidth : 20, type : Integer, read : {
                                                 int count = 0
                                                 model.hashBucket[it].each { 
@@ -187,6 +197,7 @@ class SearchTabView {
                                         tableModel(list : model.senders2) {
                                             closureColumn(header : "Sender", preferredWidth : 350, type : String, read : {it.sender.getHumanReadableName()})
                                             closureColumn(header : "Browse", preferredWidth : 20, type : Boolean, read : {it.browse})
+                                            closureColumn(header : "Feed", preferredWidth : 20, type: Boolean, read : {it.feed})
                                             closureColumn(header : "Chat", preferredWidth : 20, type : Boolean, read : {it.chat})
                                             closureColumn(header : "Comment", preferredWidth : 20, type : Boolean, read : {it.comment != null})
                                             closureColumn(header : "Certificates", preferredWidth : 20, type: Integer, read : {it.certificates})
@@ -200,6 +211,7 @@ class SearchTabView {
                                     gridLayout(rows : 1, cols : 2)
                                     panel (border : etchedBorder()) {
                                         button(text : "Browse Host", enabled : bind {model.browseActionEnabled}, browseAction)
+                                        button(text : "Subscribe", enabled : bind {model.subscribeActionEnabled}, subscribeAction)
                                         button(text : "Chat", enabled : bind{model.chatActionEnabled}, chatAction)
                                         button(text : "View Comment", enabled : bind {model.viewCommentActionEnabled}, showCommentAction)
                                         button(text : "View Certificates", enabled : bind {model.viewCertificatesActionEnabled}, viewCertificatesAction)
@@ -308,6 +320,7 @@ class SearchTabView {
             if (result == null) {
                 model.viewCommentActionEnabled = false
                 model.viewCertificatesActionEnabled = false
+                model.subscribeActionEnabled = false
                 return
             } else {
                 model.viewCommentActionEnabled = result.comment != null
@@ -326,12 +339,14 @@ class SearchTabView {
             if (row < 0) {
                 model.trustButtonsEnabled = false
                 model.browseActionEnabled = false
-                model.chatActionEnabled = false
+                model.subscribeActionEnabled = false
                 return
             } else {
                 Persona sender = model.senders[row]
                 model.browseActionEnabled = model.sendersBucket[sender].first().browse
                 model.chatActionEnabled = model.sendersBucket[sender].first().chat
+                model.subscribeActionEnabled = model.sendersBucket[sender].first().feed &&
+                    model.core.feedManager.getFeed(sender) == null
                 model.trustButtonsEnabled = true
                 model.results.clear()
                 model.results.addAll(model.sendersBucket[sender])
@@ -386,16 +401,19 @@ class SearchTabView {
             if (row < 0 || model.senders2[row] == null) {
                 model.browseActionEnabled = false
                 model.chatActionEnabled = false
+                model.subscribeActionEnabled = false
                 model.viewCertificatesActionEnabled = false
                 model.trustButtonsEnabled = false
                 model.viewCommentActionEnabled = false
                 return
             }
-            model.browseActionEnabled = model.senders2[row].browse
-            model.chatActionEnabled = model.senders2[row].chat
+            UIResultEvent e = model.senders2[row]
+            model.browseActionEnabled = e.browse
+            model.chatActionEnabled = e.chat
+            model.subscribeActionEnabled = e.feed && model.core.feedManager.getFeed(e.getSender()) == null 
             model.trustButtonsEnabled = true
-            model.viewCommentActionEnabled = model.senders2[row].comment != null
-            model.viewCertificatesActionEnabled = model.senders2[row].certificates > 0
+            model.viewCommentActionEnabled = e.comment != null
+            model.viewCertificatesActionEnabled = e.certificates > 0
         })
        
         if (settings.groupByFile)

gui/griffon-app/views/com/muwire/gui/SignView.groovy

@@ -0,0 +1,66 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonView
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+
+import javax.swing.JDialog
+import javax.swing.SwingConstants
+
+import java.awt.BorderLayout
+import java.awt.event.WindowAdapter
+import java.awt.event.WindowEvent
+
+import javax.annotation.Nonnull
+
+@ArtifactProviderFor(GriffonView)
+class SignView {
+    @MVCMember @Nonnull
+    FactoryBuilderSupport builder
+
+    def mainFrame
+    def dialog
+    def p
+    def plainTextArea
+    def signedTextArea
+    
+    void initUI() {
+        mainFrame = application.windowManager.findWindow("main-frame")
+        
+        dialog = new JDialog(mainFrame, "Sign Text", true)
+        
+        p = builder.panel {
+            borderLayout()
+            panel (constraints : BorderLayout.NORTH) {
+                label("Enter text to be signed")
+            }
+            panel (constraints : BorderLayout.CENTER) {
+                gridLayout(rows : 2, cols: 1)
+                scrollPane {
+                    plainTextArea = textArea(rows : 10, columns : 50, editable : true, lineWrap: true, wrapStyleWord : true)
+                }
+                scrollPane {
+                    signedTextArea = textArea(rows : 10, columns : 50, editable : false, lineWrap : true, wrapStyleWord : true)
+                }
+            }
+            panel (constraints : BorderLayout.SOUTH) {
+                button(text : "Sign", signAction)
+                button(text : "Copy To Clipboard", copyAction)
+                button(text : "Dismiss", closeAction)
+            }
+        }
+    }
+    
+    void mvcGroupInit(Map<String,String> args) {
+        dialog.getContentPane().add(p)
+        dialog.pack()
+        dialog.setLocationRelativeTo(mainFrame)
+        dialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
+        dialog.addWindowListener( new WindowAdapter() {
+            public void windowClosed(WindowEvent e) {
+                mvcGroup.destroy()
+            }
+        })
+        dialog.show()
+    }
+}

gui/griffon-app/views/com/muwire/gui/WatchedDirectoryView.groovy

@@ -0,0 +1,74 @@
+
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonView
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+
+import javax.swing.JDialog
+import javax.swing.SwingConstants
+import javax.swing.event.ChangeListener
+
+import java.awt.BorderLayout
+import java.awt.GridBagConstraints
+import java.awt.event.WindowAdapter
+import java.awt.event.WindowEvent
+
+import javax.annotation.Nonnull
+
+@ArtifactProviderFor(GriffonView)
+class WatchedDirectoryView {
+    @MVCMember @Nonnull
+    FactoryBuilderSupport builder
+    @MVCMember @Nonnull
+    WatchedDirectoryModel model
+    
+    def dialog
+    def p
+    def mainFrame
+    
+    def autoWatchCheckbox
+    def syncIntervalField
+
+    void initUI() {
+        mainFrame = application.windowManager.findWindow("main-frame")
+        dialog = new JDialog(mainFrame, "Watched Directory Configuration", true)
+        dialog.setResizable(false)
+        
+        p = builder.panel {
+            borderLayout()
+            panel (constraints : BorderLayout.NORTH) {
+                label("Configuration for directory " + model.directory.directory.toString())
+            }
+            panel (constraints : BorderLayout.CENTER) {
+                gridBagLayout()
+                label(text : "Auto-watch directory using operating system", constraints : gbc(gridx: 0, gridy : 0, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                autoWatchCheckbox = checkBox(selected : bind {model.autoWatch}, constraints : gbc(gridx: 1, gridy : 0, anchor : GridBagConstraints.LINE_END))
+                label(text : "Directory sync frequency (seconds, 0 means never)", enabled : bind {!model.autoWatch}, constraints : gbc(gridx: 0, gridy : 1, anchor : GridBagConstraints.LINE_START, weightx: 100))
+                syncIntervalField = textField(text : bind {model.syncInterval}, columns: 4, enabled : bind {!model.autoWatch}, 
+                    constraints: gbc(gridx: 1, gridy : 1, anchor : GridBagConstraints.LINE_END, insets : [0,10,0,0]))
+            }
+            panel (constraints : BorderLayout.SOUTH) {
+                button(text : "Save", saveAction)
+                button(text : "Cancel", cancelAction)
+            }
+        }
+    }
+        
+    void mvcGroupInit(Map<String,String> args) {
+        autoWatchCheckbox.addChangeListener({e ->
+            model.autoWatch = autoWatchCheckbox.model.isSelected()
+        } as ChangeListener)
+        
+        dialog.getContentPane().add(p)
+        dialog.pack()
+        dialog.setLocationRelativeTo(mainFrame)
+        dialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
+        dialog.addWindowListener(new WindowAdapter() {
+            public void windowClosed(WindowEvent e) {
+                mvcGroup.destroy()
+            }
+        })
+        dialog.show()
+    }
+}

gui/src/main/groovy/com/muwire/gui/DownloaderComparator.groovy

@@ -6,8 +6,8 @@ class DownloaderComparator implements Comparator<Downloader>{
 
     @Override
     public int compare(Downloader o1, Downloader o2) {
-        double d1 = o1.donePieces() * 1.0 / o1.nPieces
-        double d2 = o2.donePieces() * 1.0 / o2.nPieces
+        double d1 = o1.donePieces().toDouble() / o1.nPieces
+        double d2 = o2.donePieces().toDouble() / o2.nPieces
         return Double.compare(d1, d2);
     }
 }

host-cache/build.gradle

@@ -6,4 +6,5 @@ dependencies {
     compile "net.i2p:i2p:${i2pVersion}"
     testCompile 'org.junit.jupiter:junit-jupiter-api:5.4.2'
     testCompile 'junit:junit:4.12'
+    testCompile 'org.codehaus.groovy:groovy-all:2.4.15'
 }

host-cache/logging/logging.properties

@@ -0,0 +1,62 @@
+############################################################
+#       Default Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.  
+# For example java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+#       Global properties
+############################################################
+
+# "handlers" specifies a comma separated list of log Handler 
+# classes.  These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+handlers= java.util.logging.FileHandler
+
+# To also add the FileHandler, use the following line instead.
+#handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers.  For any given facility this global level
+# can be overriden by a facility specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+.level= INFO
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+# default file output is in user's home directory.
+java.util.logging.FileHandler.pattern = hostcache.log
+java.util.logging.FileHandler.limit = 5000000
+java.util.logging.FileHandler.count = 1
+java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
+
+# Limit the message that are printed on the console to INFO and above.
+java.util.logging.ConsoleHandler.level = INFO
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+# Example to customize the SimpleFormatter output format 
+# to print one-line log message like this:
+#     <level>: <log message> [<date/time>]
+#
+#java.util.logging.SimpleFormatter.format=%4$s: %5$s [%1$tc]%n
+
+java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$s %2$s %5$s %6$s %n
+
+############################################################
+# Facility specific properties.
+# Provides extra control for each logger.
+############################################################
+
+# For example, set the com.xyz.foo logger to only log SEVERE
+# messages:
+com.xyz.foo.level = SEVERE
+net.i2p.client.streaming.impl.level = SEVERE

host-cache/scripts/count_total.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+  
+import os,sys,json
+
+if len(sys.argv) < 2 :
+    print("This script counts unique hosts in the MuWire network",file = sys.stderr)
+    print("Pass the prefix of the files to analyse.  For example:",file = sys.stderr)
+    print("\"20200427\" will count unique hosts on 27th of April 2020",file = sys.stderr)
+    print("\"202004\" will count unique hosts during all of April 2020",file = sys.stderr)
+    sys.exit(1)
+
+day = sys.argv[1]
+files = os.listdir(".")
+files = [x for x in files if x.startswith(day)]
+
+hosts = set()
+
+for f in files:
+    for line in open(f):
+        host = json.loads(line)
+        hosts.add(host["destination"])
+
+print(len(hosts))

host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy

@@ -40,12 +40,13 @@ class Crawler {
         try {
             uuid = UUID.fromString(pong.uuid)
         } catch (IllegalArgumentException bad) {
+            log.log(Level.WARNING,"couldn't parse uuid",bad)
             hostPool.fail(host)
             return
         }
 
         if (!uuid.equals(currentUUID)) {
-            log.info("uuid mismatch")
+            log.warning("uuid mismatch $uuid expected $currentUUID")
             hostPool.fail(host)
             return
         }
@@ -75,11 +76,12 @@ class Crawler {
     }
 
     synchronized def startCrawl() {
+        currentUUID = UUID.randomUUID()
+        log.info("starting new crawl with uuid $currentUUID inFlight ${inFlight.size()}")
         if (!inFlight.isEmpty()) {
             inFlight.values().each { hostPool.fail(it) }
             inFlight.clear()
         }
-        currentUUID = UUID.randomUUID()
         hostPool.getUnverified(parallel).each {
             inFlight.put(it.destination, it)
             pinger.ping(it, currentUUID)