Index: /data_recovery/trunk/README
===================================================================
--- /data_recovery/trunk/README	(revision 30)
+++ /data_recovery/trunk/README	(revision 30)
@@ -0,0 +1,2 @@
+Use salvage_data.py to read data off the disk, and stream a log of the data.
+Then use recover_log.py to turn that back into a disk image like you would have gotten with dd if there weren't problems, and also a more compact log file.
Index: /data_recovery/trunk/recover_log.py
===================================================================
--- /data_recovery/trunk/recover_log.py	(revision 30)
+++ /data_recovery/trunk/recover_log.py	(revision 30)
@@ -0,0 +1,172 @@
+#!/usr/bin/python
+"""Recovers a disk image from a log file generated by salvage_data.py.
+"""
+import sys
+
+def coalesce_extents(one, two):
+    """ASSUMES the extents overlap."""
+    return (min(one[0], two[0]), max(one[1], two[1]))
+
+def extents_overlap(one, two):
+    """returns True if they touch or overlap in some way, False if they don't.
+    """
+    return one[0] <= two[0] <= one[1] or \
+        one[0] <= two[1] <= one[1] or \
+        two[0] <= one[0] <= two[1] or \
+        two[0] <= one[1] <= two[1]
+
+class Extents(object):
+    """Tracks the extents that have been covered by the log."""
+    def __init__(self):
+        self.extents = []
+
+    def add(self, offset, length):
+        """Adds an extent at offset of the given length."""
+        self.add_extent((offset, offset+length))
+
+    def add_extent(self, extent):
+        """Adds an extent."""
+        start, end = extent
+        if start == end: # nothing to add
+            return
+
+        # find any existing extent that overlaps with the one being added.
+        for i in range(len(self.extents)):
+            current_extent = self.extents[i]
+            if extents_overlap(extent, current_extent):
+                self.extents.pop(i)
+                new_extent = coalesce_extents(current_extent, extent)
+                self.add_extent(new_extent)
+                return
+
+        # No coalescing needed
+        self.extents.append(extent)
+
+    def bytes_covered(self):
+        """Returns the number of bytes covered by the extents."""
+        return sum([b-a for a, b in self.extents])
+
+    def byte_range(self):
+        """Returns a tuple of the starting and ending offsets covered by these
+        extents.
+        """
+        self.extents.sort()
+        return (self.extents[0][0], self.extents[-1][1])
+
+    def __str__(self):
+        start, end = self.byte_range()
+        covered = self.bytes_covered()
+        return ' '.join([repr(e) for e in self.extents]) + \
+            "\n%s of %s bytes covered (%s remain)" % (covered, end-start,
+            end-start-covered)
+
+def report_log(good, bad):
+    """let the user know what extents have been accounted for"""
+    sys.stderr.write("Good extents: %s\nBad extents: %s\n" % (good, bad))
+
+def write_image_from_log(log, image):
+    """Reads from the log file object, and writes the data to the image file
+    object.
+    """
+    good_extents = Extents()
+    bad_extents = Extents()
+    try:
+        while True:
+            meta = log.readline().split()
+            if not meta:
+                break
+            if meta[0] == 'D': # data
+                offset = long(meta[1])
+                length = long(meta[2])
+                data = log.read(length)
+                if len(data) != length:
+                    raise Exception("Short line: %s of %s bytes at offset %s" \
+                        % (len(data), length, offset))
+                log.read(1) # the extra newline
+
+                sys.stderr.write("writing %s bytes at %s\n" % (length, offset))
+                image.seek(offset)
+                image.write(data)
+                good_extents.add(offset, length)
+            elif meta[0] == 'E':
+                offset = long(meta[1])
+                if len(meta) > 2:
+                    length = long(meta[2])
+                else:
+                    length = 1
+                sys.stderr.write("skipping %s bad bytes at %s\n" % (length,
+                    offset))
+                bad_extents.add(offset, length)
+            else:
+                raise Exception("Invalid line: %r" % (meta,))
+    except:
+        report_log(good_extents, bad_extents)
+        raise
+
+    report_log(good_extents, bad_extents)
+    return good_extents, bad_extents
+
+def write_log_from_image(image, out, good_extents, bad_extents):
+    """Write out a concise log file with the same information as the input
+    file.
+    """
+    max_extent_size = 10*1024**2
+
+    extents = [(s, e, 'D') for s, e in good_extents.extents] + \
+              [(s, e, 'E') for s, e in bad_extents.extents]
+    extents.sort()
+    for start, end, state in extents:
+        if state == 'E':
+            out.write("E %s %s\n" % (start, end - start))
+            out.flush()
+        elif state == 'D':
+            offset = start
+            while offset < end:
+                image.seek(offset)
+                chunk = min(max_extent_size, end-offset)
+                data = image.read(chunk)
+                if len(data) != chunk:
+                    raise Exception("Short read from image file")
+                out.write("D %s %s\n%s\n" % (offset, chunk, data))
+                out.flush()
+                offset += chunk
+        else:
+            raise Exception("INTERNAL ERROR: Invalid state \"%s\"" % state)
+
+
+def usage(out):
+    """outputs help message"""
+    out.write("Syntax Error: %s [-l] <imagefilename>\n"
+            "Requires the file to which to write the image, and reads the "
+            "recovery log from stdin.\n"
+            "/dev/null may be specified as the image filename to just show "
+            "summary information.\n"
+    )
+
+def main(args):
+    """Reads log from standard in, writes an image to the image file"""
+    if len(args) < 1:
+        usage(sys.stderr)
+        sys.exit(1)
+
+    output_log = False
+    if args[0] == '-l':
+        output_log = True
+        if len(args) != 2:
+            usage(sys.stderr)
+            sys.exit(1)
+        filename = args[1]
+    else:
+        if len(args) != 1:
+            usage(sys.stderr)
+            sys.exit(1)
+        filename = args[0]
+
+    image = open(filename, 'w')
+
+    good, bad = write_image_from_log(sys.stdin, image)
+    if output_log:
+        write_log_from_image(open(filename, 'r'), sys.stdout, good, bad)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
Index: /data_recovery/trunk/salvage_data.py
===================================================================
--- /data_recovery/trunk/salvage_data.py	(revision 30)
+++ /data_recovery/trunk/salvage_data.py	(revision 30)
@@ -0,0 +1,114 @@
+#!/usr/bin/python
+"""salvage_data
+
+This is a tool for recovering data from a partially failing drive.  It will
+start at the beginning of the device, and start reading sequentially until it
+hits a problem, and will then split the remaining work to avoid bad sections as
+long as possible.
+"""
+import sys
+
+class WorkQueue(object):
+    """Queue for the extents left to attempt to read."""
+    def __init__(self, start, end):
+        self.queue = [(start, end)]
+
+    def is_empty(self):
+        """Lets us know when we are done"""
+        return not self.queue
+
+    def pop(self):
+        """Returns a (start, end) tuple of the next section to attempt"""
+        return self.queue.pop()
+
+    def push(self, start, end, split=False):
+        """Adds the start, end section to the queue, splitting if told to"""
+        if start >= end:
+            # nothing to queue
+            return
+        if split:
+            middle = (start + end) / 2
+            if start < middle:
+                self.queue.insert(0, (start, middle))
+            if middle < end:
+                self.queue.append((middle, end))
+        else:
+            self.queue.append((start, end))
+
+def recover(drive, start, end, output):
+    """Recovers data from the provided file object within the given start and
+    end, and writes the log to the output file object.
+    """
+    extent_size = 1024*1024 # Size of blocks to attempt to read
+    # queue of data ranges to try to recover
+    queue = WorkQueue(start, end)
+
+    while not queue.is_empty():
+        start, end = queue.pop()
+        if end - start < extent_size:
+            chunk = end - start
+        else:
+            chunk = extent_size
+        try:
+            drive.seek(start)
+            data = drive.read(chunk)
+            error = False
+        except IOError:
+            error = True
+
+        if error:
+            # error reading at this offset
+            output.write("E %s\n" % start)
+            output.flush()
+            queue.push(start+1, end, True)
+        elif not data:
+            # error reading at this offset; I think this only happened due to a
+            # software bug
+            output.write("S %s\n" % start)
+            output.flush()
+            queue.push(start+1, end, True)
+        else:
+            output.write("D %s %s\n%s\n" % (start, len(data), data))
+            output.flush()
+            # if we had a short read, probably an error at the end of this, so
+            # split the extent
+            queue.push(start+len(data), end, len(data)<chunk)
+
+def main(args):
+    """Takes the device to read as the first argument.  May optionally specify
+    the start and end offsets to recover.
+    """
+    if len(args) < 1:
+        sys.stderr.write("Syntax error: %s <inputfile> [start [end]]\n"
+            "Reads from the specified input file (device)\n"
+            "Optionally starts at the specified start offset and ends at the "
+            "specified end offset.\n"
+            "If not specified, start defaults to 0, and end defaults to the "
+            "end of the device.\n"
+        )
+        sys.exit(1)
+
+    device = args[0]
+
+    # determine device size
+    drive = open(device, 'r')
+    drive.seek(0, 2)
+    end = drive.tell()
+
+    # see if the user has specified a range to recover
+    start = 0
+    try:
+        start = int(args[1])
+        end = int(args[2])
+    except IndexError:
+        pass
+
+    sys.stderr.write("Reading %s from %s to %s\n" % (device, start, end))
+    sys.stderr.flush()
+
+    output = sys.stdout
+
+    recover(drive, start, end, output)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
Index: /mergebot/branches/ticket-1/mergebot/SvnOps.py
===================================================================
--- /mergebot/branches/ticket-1/mergebot/SvnOps.py	(revision 20)
+++ /mergebot/branches/ticket-1/mergebot/SvnOps.py	(revision 30)
@@ -69,5 +69,5 @@
     skipped_regex = re.compile("Skipped.* '(.*)'", re.M)
     start_rev, end_rev = revision_range
-    pipe = os.popen("cd %s && svn merge --revision %s:%s %s . 2>>%s" % \
+    pipe = os.popen("cd %s && svn merge --non-interactive --revision %s:%s %s . 2>>%s" % \
         (workingdir, start_rev, end_rev, from_url, logfile))
     output = pipe.readlines()
@@ -79,4 +79,10 @@
             filename = skipped_regex.findall(line)[0]
             status = "C"
+        elif line.startswith('--- Merging '):
+            continue # ignore this line for now
+        elif line.startswith('Summary of conflicts:'):
+            continue # ignore this line for now
+        elif line.startswith('  Text conflicts:'):
+            continue # ignore this line for now
         else:
             assert line[4] == ' ', "Unexpected output from svn merge " \
Index: /mergebot/trunk/README.txt
===================================================================
--- /mergebot/trunk/README.txt	(revision 20)
+++ /mergebot/trunk/README.txt	(revision 30)
@@ -3,5 +3,5 @@
 $ svn co <url> mergebot-0.11
 $ cd mergebot-0.11
-$ ./rpm/makerpm
+$ python setup.py bdist_rpm
 $ su -c "rpm --install dist/TracMergeBot*.noarch.rpm"
 # Create the mergebot work area
Index: /mergebot/trunk/mergebot/Actor.py
===================================================================
--- /mergebot/trunk/mergebot/Actor.py	(revision 20)
+++ /mergebot/trunk/mergebot/Actor.py	(revision 30)
@@ -1,5 +1,9 @@
+"""Base class for mergebot actors that do the various kinds of tasks
+"""
 import os
+from mergebot import SvnOps
 
 class Actor(object):
+    """Base class for mergebot actors"""
     def __init__(self, work_dir, repo_url, repo_dir, ticket, component,
                  version, summary, requestor):
@@ -23,8 +27,10 @@
 
     def logfilename(self):
+        """Returns the absolute path of the logfile for this ticket"""
         return os.path.abspath(os.path.join(os.path.dirname(self.work_dir),
             'ticket-%s.log' % self.ticket))
 
     def public_url(self):
+        """Returns the public URL for this component"""
         return '%s/%s' % (self.repo_url, self.component)
 
@@ -35,4 +41,5 @@
 
     def local_url(self):
+        """Returns the local URL for this component"""
         return 'file://%s/%s' % (self.repo_dir, self.component)
 
@@ -48,4 +55,5 @@
 
     def version_subdir(self):
+        """Returns the subdirectory name for the version"""
         if self.version == 'trunk':
             subdir = 'trunk'
@@ -58,2 +66,15 @@
         return subdir
 
+    def check_required_directories(self):
+        """Make sure the various urls we require do exist"""
+        if not SvnOps.does_url_exist(self.local_url()):
+            return 'Component %s does not exist in the repository.' \
+                % self.component
+        if not SvnOps.does_url_exist(self.local_url() + '/branches'):
+            return 'No directory in which to create branches for ' \
+                'component %s in the repository.' % self.component
+        if not SvnOps.does_url_exist(self.baseline_local_url()):
+            return 'Version %s for component %s does not exist in the ' \
+                'repository.' % (self.version, self.component)
+        return None
+
Index: /mergebot/trunk/mergebot/BranchActor.py
===================================================================
--- /mergebot/trunk/mergebot/BranchActor.py	(revision 20)
+++ /mergebot/trunk/mergebot/BranchActor.py	(revision 30)
@@ -2,5 +2,4 @@
 """Module for creating new branches for tickets"""
 
-import os
 import time
 
@@ -20,14 +19,7 @@
 
         # Make sure the various urls we require do exist
-        if not SvnOps.get_branch_info(self.local_url(), logfile):
-            comment = 'Component %s does not exist in the repository.' \
-                % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.local_url() + '/branches', logfile):
-            comment = 'No directory in which to create branches for component %s in the repository.' % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.baseline_local_url(), logfile):
-            comment = 'Version %s for component %s does not exist in the repository.' % (self.version, self.component)
-            return results, comment, False
+        problems = self.check_required_directories()
+        if problems:
+            return results, problems, False
 
         commit_header = 'Ticket #%s: %s' % (self.ticket, self.summary)
@@ -35,5 +27,5 @@
         # Delete the branch if it already exists.  This can happen if the branch
         # was merged, but we're still working on it.
-        if SvnOps.get_branch_info(self.branch_local_url(), logfile):
+        if SvnOps.does_url_exist(self.branch_local_url()):
             # This branch already exists.
             commit_message = "\n".join([commit_header,
@@ -59,9 +51,11 @@
             comment = 'Failed to create branch.'
             return results, comment, False
+
         results['mergebotstate'] = 'branched'
         comment = '\n'.join([
             'Created branch from %s for %s.' % (self.version, self.requestor),
             '',
-            'Browse branch [source:%s/branches/ticket-%s source code] and [log:%s/branches/ticket-%s commit log].' % 
+            'Browse branch [source:%s/branches/ticket-%s source code] and ' \
+                '[log:%s/branches/ticket-%s commit log].' % 
                 (self.component, self.ticket, self.component, self.ticket),
             '',
Index: /mergebot/trunk/mergebot/CheckMergeActor.py
===================================================================
--- /mergebot/trunk/mergebot/CheckMergeActor.py	(revision 20)
+++ /mergebot/trunk/mergebot/CheckMergeActor.py	(revision 30)
@@ -28,16 +28,7 @@
 
         # Make sure the various urls we require do exist
-        if not SvnOps.get_branch_info(self.local_url(), logfile):
-            comment = 'Component %s does not exist in the repository.' \
-                % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.local_url() + '/branches', logfile):
-            comment = 'No directory in which to create branches for ' \
-                'component %s in the repository.' % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.baseline_local_url(), logfile):
-            comment = 'Version %s for component %s does not exist in the ' \
-                'repository.' % (self.version, self.component)
-            return results, comment, False
+        problems = self.check_required_directories()
+        if problems:
+            return results, problems, False
 
         branch_info = SvnOps.get_branch_info(self.branch_local_url(), logfile)
Index: /mergebot/trunk/mergebot/MergeActor.py
===================================================================
--- /mergebot/trunk/mergebot/MergeActor.py	(revision 20)
+++ /mergebot/trunk/mergebot/MergeActor.py	(revision 30)
@@ -26,18 +26,12 @@
 
         # Make sure the various urls we require do exist
-        if not SvnOps.get_branch_info(self.local_url(), logfile):
-            comment = 'Component %s does not exist in the repository.' \
-                % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.local_url() + '/branches', logfile):
-            comment = 'No directory in which to create branches for component %s in the repository.' % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.baseline_local_url(), logfile):
-            comment = 'Version %s for component %s does not exist in the repository.' % (self.version, self.component)
-            return results, comment, False
+        problems = self.check_required_directories()
+        if problems:
+            return results, problems, False
 
         rev_info = SvnOps.get_branch_info(self.branch_local_url(), logfile)
         if not rev_info:
-            comment = 'Branch for ticket %s does not exist in the repository.' % (self.ticket)
+            comment = 'Branch for ticket %s does not exist in the repository.' \
+                % (self.ticket)
             return results, comment, False
         startrev, endrev = rev_info
Index: /mergebot/trunk/mergebot/RebranchActor.py
===================================================================
--- /mergebot/trunk/mergebot/RebranchActor.py	(revision 20)
+++ /mergebot/trunk/mergebot/RebranchActor.py	(revision 30)
@@ -36,16 +36,7 @@
 
         # Make sure the various urls we require do exist
-        if not SvnOps.get_branch_info(self.local_url(), logfile):
-            comment = 'Component %s does not exist in the repository.' \
-                % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.local_url() + '/branches', logfile):
-            comment = 'No directory in which to create branches for ' \
-                'component %s in the repository.' % self.component
-            return results, comment, False
-        if not SvnOps.get_branch_info(self.baseline_local_url(), logfile):
-            comment = 'Version %s for component %s does not exist in the ' \
-                'repository.' % (self.version, self.component)
-            return results, comment, False
+        problems = self.check_required_directories()
+        if problems:
+            return results, problems, False
 
         rev_info = SvnOps.get_branch_info(self.branch_local_url(), logfile)
@@ -97,7 +88,4 @@
                 "Rebranch internal error: Unable to recreate the branch.  %s" \
                     % (instructioncomment, )),
-            (lambda x: SvnOps.checkout(self.branch_local_url(), workingcopy, x),
-                "Rebranch internal error: Unable to get working copy.  %s" % \
-                    (instructioncomment, )),
         )
         for cmd, error_comment in commanderrors:
@@ -108,4 +96,27 @@
                 results['mergebotstate'] = 'rebranchfailed'
                 return results, error_comment, False
+
+        # Check to see if there have been no commits on the branch.  If there
+        # have been no commits, we know we don't need to merge anything.
+        if startrev == endrev:
+            ticket_message = "\n".join([
+                "Rebranched from %s for %s." % (self.version,
+                                                self.requestor),
+                "There were no changes to commit to the branch since there "
+                    "were no commits on the branch.",
+                "You will need to update your working copy.",
+            ])
+            results['mergebotstate'] = 'branched'
+            return results, ticket_message, True
+
+        retval = SvnOps.checkout(self.branch_local_url(), workingcopy, logfile)
+        if retval:
+            error_comment = \
+                "Rebranch internal error: Unable to get working copy.  " + \
+                instructioncomment
+            if os.path.exists(workingcopy):
+                shutil.rmtree(workingcopy)
+            results['mergebotstate'] = 'rebranchfailed'
+            return results, error_comment, False
 
         # On success, we're in the same state as if we had just branched.
@@ -135,4 +146,5 @@
                     "commit your work to the branch.",
             ]) 
+            success = False # the rebranch failed because there were conflicts
         else: # No conflicts, do the commit.
             mergemessage = "\n".join([
Index: /mergebot/trunk/mergebot/SvnOps.py
===================================================================
--- /mergebot/trunk/mergebot/SvnOps.py	(revision 20)
+++ /mergebot/trunk/mergebot/SvnOps.py	(revision 30)
@@ -8,4 +8,5 @@
 import time
 import re
+import subprocess
 
 def shell_quote(string):
@@ -24,4 +25,20 @@
     """Given a log entry split out of svn log, return its revision number"""
     return int(logentry.split()[0][1:])
+
+def does_url_exist_14(url):
+    """Given a subversion url return true if it exists, false otherwise."""
+    return not subprocess.call(['svn', 'log', '--limit=1', '--non-interactive',
+                                url],
+                    stdout=open('/dev/null', 'w'),
+                    stderr=open('/dev/null', 'w'))
+
+def does_url_exist_15(url):
+    """Given a subversion url return true if it exists, false otherwise."""
+    return not subprocess.call(['svn', 'ls', '--depth', 'empty',
+                                '--non-interactive', url],
+                    stdout=open('/dev/null', 'w'),
+                    stderr=open('/dev/null', 'w'))
+
+does_url_exist=does_url_exist_14 # default to most compatible form for now
 
 def get_branch_info(url, logfile):
Index: /mergebot/trunk/setup.py
===================================================================
--- /mergebot/trunk/setup.py	(revision 20)
+++ /mergebot/trunk/setup.py	(revision 30)
@@ -5,5 +5,5 @@
 setup(
     name = "TracMergeBot",
-    version = "0.11",
+    version = "0.11.1",
     author = "Eli Carter",
     author_email = "eli.carter@retracile.net",
