diff --git a/contained/bin/fake-dns.py b/contained/bin/fake-dns.py
index 1f0878d..4d23f89 100644
--- a/contained/bin/fake-dns.py
+++ b/contained/bin/fake-dns.py
@@ -63,6 +63,9 @@ class UnixSocketServer(socketserver.ThreadingMixIn, socketserver.UnixStreamServe
     """
 
     def __init__(self, socket_path, request_handler_class, bind_and_activate=True):
+        self._socket_path = socket_path
+        # Remove old socket if it's not being used
+        self._rm_socket()
         super().__init__(socket_path, request_handler_class, bind_and_activate)
         self.thread = threading.Thread(target=self.serve_forever)
         self.thread.daemon = True
@@ -73,6 +76,17 @@ class UnixSocketServer(socketserver.ThreadingMixIn, socketserver.UnixStreamServe
     def stop(self):
         self.shutdown()
         self.server_close()
+        # Attempt to remove socket before exiting
+        self._rm_socket(True)
+
+    def _rm_socket(self, log_error=False):
+        try:
+            os.unlink(self._socket_path)
+        except Exception as e:
+            if log_error:
+                logging.getLogger("UnixSocketServer").warn(
+                    "Couldn't remove unix socket: %s", e
+                )
 
     def isAlive(self):
         return self.thread.isAlive()