Make batch mode actually work and add man page documentation. From Jos Backus.

NEWS

@@ -7,5 +7,7 @@ rsync 2.5.3 (not released yet)
       what was historically used so as not to break scripts that try
       to parse the output.
 
-    * Added --no-whole-file and --no-blocking-io options
+    * Added --no-whole-file and --no-blocking-io options (Dave Dykstra)
 
+    * Made the --write-batch and --read-batch options actually work
+	and added documentation in the man page (Jos Backus)

errcode.h

@@ -26,6 +26,7 @@
 #define RERR_PROTOCOL   2       /* protocol incompatibility */
 #define RERR_FILESELECT 3       /* errors selecting input/output files, dirs */
 #define RERR_UNSUPPORTED 4       /* requested action not supported */
+#define RERR_STARTCLIENT 5      /* error starting client-server protocol */
 
 #define RERR_SOCKETIO   10      /* error in socket IO */
 #define RERR_FILEIO     11      /* error in file IO */

log.c

@@ -41,6 +41,7 @@ struct {
 	{ RERR_PROTOCOL   , "protocol incompatibility" }, 
 	{ RERR_FILESELECT , "errors selecting input/output files, dirs" }, 
 	{ RERR_UNSUPPORTED, "requested action not supported" }, 
+	{ RERR_STARTCLIENT, "error starting client-server protocol" }, 
 	{ RERR_SOCKETIO   , "error in socket IO" }, 
 	{ RERR_FILEIO     , "error in file IO" }, 
 	{ RERR_STREAMIO   , "error in rsync protocol data stream" }, 
@@ -559,20 +560,16 @@ void log_exit(int code, const char *file, int line)
 	}
 }
 
-
-
-
-/* log the incoming transfer of a file for interactive use, this
-   will be called at the end where the client was run 
-   
-   it i called when a file starts to be transferred
-*/
+/*
+ * Log the incoming transfer of a file for interactive use,
+ * this will be called at the end where the client was run.
+ * Called when a file starts to be transferred.
+ */
 void log_transfer(struct file_struct *file, const char *fname)
 {
 	extern int verbose;
 
 	if (!verbose) return;
 
-	rprintf(FINFO,"%s\n", fname);
+	rprintf(FINFO, "%s\n", fname);
 }
-

main.c

@@ -177,7 +177,7 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int
 	extern int blocking_io;
 	extern int read_batch;
 
-	if (!read_batch && !local_server) { /* dw -- added read_batch */
+	if (!read_batch && !local_server) {
 		if (!cmd)
 			cmd = getenv(RSYNC_RSH_ENV);
 		if (!cmd)
@@ -230,7 +230,7 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int
 
 	if (local_server) {
 		if (read_batch)
-		    create_flist_from_batch();
+		    create_flist_from_batch(); /* sets batch_flist */
 		ret = local_child(argc, args, f_in, f_out);
 	} else {
 		ret = piped_child(args,f_in,f_out);
@@ -443,8 +443,8 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[])
 	extern int am_daemon;
 	extern int module_id;
 	extern int am_sender;
-	extern int read_batch;   /* dw */
-	extern struct file_list *batch_flist;  /* dw */
+	extern int read_batch;
+	extern struct file_list *batch_flist;
 
 	if (verbose > 2)
 		rprintf(FINFO,"server_recv(%d) starting pid=%d\n",argc,(int)getpid());
@@ -470,7 +470,7 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[])
 	if (delete_mode && !delete_excluded)
 		recv_exclude_list(f_in);
 
-	if (read_batch) /*  dw  */
+	if (read_batch)
 	    flist = batch_flist;
 	else
 	    flist = recv_file_list(f_in);
@@ -497,7 +497,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
 	extern int cvs_exclude;
 	extern int am_sender;
 	extern int remote_version;
-	extern int read_batch; /* dw */
+	extern int read_batch;
 
 	setup_protocol(f_out, f_in);
 
@@ -508,7 +508,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
 		io_start_multiplex_out(f_out);
 
 	if (am_sender) {
-		if (!read_batch) { /* dw */
+		if (!read_batch) {
 		    recv_exclude_list(f_in);
 		    if (cvs_exclude)
 			add_cvs_excludes();
@@ -527,19 +527,19 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
  */
 int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 {
-	struct file_list *flist;
+	struct file_list *flist = NULL;
 	int status = 0, status2 = 0;
 	char *local_name = NULL;
 	extern int am_sender;
 	extern int remote_version;
 	extern pid_t cleanup_child_pid;
-	extern int write_batch; /* dw */
-	extern int read_batch; /* dw */
-	extern struct file_list *batch_flist; /*  dw */
+	extern int write_batch;
+	extern int read_batch;
+	extern struct file_list *batch_flist;
 
 	cleanup_child_pid = pid;
 	if (read_batch)
-	    flist = batch_flist;  /* dw */
+	    flist = batch_flist;
 
 	set_nonblocking(f_in);
 	set_nonblocking(f_out);
@@ -582,7 +582,7 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 		list_only = 1;
 	}
 	
-	if (!write_batch) /* dw */
+	if (!write_batch)
 	    send_exclude_list(f_out);
 	
 	flist = recv_file_list(f_in);
@@ -658,6 +658,7 @@ static int start_client(int argc, char *argv[])
 	extern char *shell_cmd;
 	extern int rsync_port;
 	extern int whole_file;
+	extern int write_batch;
 	extern int read_batch;
 	int rc;
 
@@ -685,7 +686,7 @@ static int start_client(int argc, char *argv[])
 		return start_socket_client(host, path, argc-1, argv+1);
 	}
 
-	if (!read_batch) { /* dw */
+	if (!read_batch) {
 	    p = find_colon(argv[0]);
 
 	if (p) {
@@ -711,8 +712,11 @@ static int start_client(int argc, char *argv[])
 		p = find_colon(argv[argc-1]);
 		if (!p) {
 			local_server = 1;
-			/* disable "rsync algorithm" when both sides local */
-			if (whole_file == -1)
+			/*
+			 * disable "rsync algorithm" when both sides local,
+			 * except when creating a batch update
+			 */
+			if (!write_batch && whole_file == -1)
 				whole_file = 1;
 		} else if (p[1] == ':') {
 			*p = 0;
@@ -735,9 +739,9 @@ static int start_client(int argc, char *argv[])
 		argc--;
 	}
 	} else {
-	    am_sender = 1;  /*  dw */
-	    local_server = 1;  /* dw */
-	    shell_path = argv[argc-1];  /* dw */
+	    am_sender = 1;
+	    local_server = 1;
+	    shell_path = argv[argc-1];
 	}
 
 	if (shell_machine) {
@@ -802,13 +806,11 @@ int main(int argc,char *argv[])
 	extern int am_daemon;
 	extern int am_server;
 	int ret;
-	extern int read_batch;   /*  dw */
-	extern int write_batch;  /*  dw */
-	extern char *batch_ext;   /*  dw */
-	int orig_argc;  /* dw */
+	extern int write_batch;
+	int orig_argc;
 	char **orig_argv;
 
-	orig_argc = argc;   /* dw */
+	orig_argc = argc;
 	orig_argv = argv;
 
 	signal(SIGUSR1, sigusr1_handler);
@@ -847,15 +849,10 @@ int main(int argc,char *argv[])
 	   that implement getcwd that way "pwd" can't be found after chroot. */
 	push_dir(NULL,0);
 
-	if (write_batch) { /* dw */
-	    create_batch_file_ext();
+	if (write_batch && !am_server) {
 	    write_batch_argvs_file(orig_argc, orig_argv);
 	}
 
-	if (read_batch) { /* dw */
-	    set_batch_file_ext(batch_ext);
-	}
-
 	if (am_daemon) {
 		return daemon_main();
 	}
@@ -882,7 +879,9 @@ int main(int argc,char *argv[])
 	}
 
 	ret = start_client(argc, argv);
-	exit_cleanup(ret);
+	if (ret == -1) 
+	    exit_cleanup(RERR_STARTCLIENT);
+	else
+	    exit_cleanup(ret);
 	return ret;
 }
-

options.c

@@ -87,9 +87,8 @@ int default_af_hint = AF_INET;	/* Must use IPv4 */
  * or under Unix process-monitors. **/
 int no_detach = 0;
 
-
-int read_batch=0;
-int write_batch=0;
+int write_batch = 0;
+int read_batch = 0;
 
 char *backup_suffix = BACKUP_SUFFIX;
 char *tmpdir = NULL;
@@ -107,7 +106,7 @@ int quiet = 0;
 int always_checksum = 0;
 int list_only = 0;
 
-char *batch_ext = NULL;
+char *batch_prefix = NULL;
 
 static int modify_window_set;
 
@@ -244,8 +243,8 @@ void usage(enum logcode F)
   rprintf(F,"     --log-format=FORMAT     log file transfers using specified format\n");  
   rprintf(F,"     --password-file=FILE    get password from FILE\n");
   rprintf(F,"     --bwlimit=KBPS          limit I/O bandwidth, KBytes per second\n");
-  rprintf(F,"     --read-batch=EXT        read batch file\n");
-  rprintf(F,"     --write-batch           write batch file\n");
+  rprintf(F,"     --write-batch=PREFIX    write batch fileset starting with PREFIX\n");
+  rprintf(F,"     --read-batch=PREFIX     read batch fileset starting with PREFIX\n");
   rprintf(F," -h, --help                  show this help screen\n");
 #ifdef INET6
   rprintf(F," -4                          prefer IPv4\n");
@@ -339,8 +338,8 @@ static struct poptOption long_options[] = {
   {"address",          0,  POPT_ARG_STRING, &bind_address, 0},
   {"backup-dir",       0,  POPT_ARG_STRING, &backup_dir},
   {"hard-links",      'H', POPT_ARG_NONE,   &preserve_hard_links},
-  {"read-batch",       0,  POPT_ARG_STRING, &batch_ext, OPT_READ_BATCH},
-  {"write-batch",      0,  POPT_ARG_NONE,   &write_batch},
+  {"read-batch",       0,  POPT_ARG_STRING, &batch_prefix, OPT_READ_BATCH},
+  {"write-batch",      0,  POPT_ARG_STRING, &batch_prefix, OPT_WRITE_BATCH},
 #ifdef INET6
   {0,		      '4', POPT_ARG_VAL,    &default_af_hint,   AF_INET },
   {0,		      '6', POPT_ARG_VAL,    &default_af_hint,   AF_INET6 },
@@ -523,8 +522,13 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 			keep_partial = 1;
 			break;
 
+		case OPT_WRITE_BATCH:
+			/* popt stores the filename in batch_prefix for us */
+			write_batch = 1;
+			break;
+
 		case OPT_READ_BATCH:
-			/* The filename is stored in batch_ext for us by popt */
+			/* popt stores the filename in batch_prefix for us */
 			read_batch = 1;
 			break;
 
@@ -540,6 +544,22 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 		}
 	}
 
+	if (write_batch && read_batch) {
+	    snprintf(err_buf,sizeof(err_buf),
+		"write-batch and read-batch can not be used together\n");
+	    rprintf(FERROR,"ERROR: write-batch and read-batch"
+		" can not be used together\n");
+	    return 0;
+	}
+
+	if (do_compression && (write_batch || read_batch)) {
+	    snprintf(err_buf,sizeof(err_buf),
+		"compress can not be used with write-batch or read-batch\n");
+	    rprintf(FERROR,"ERROR: compress can not be used with"
+		"  write-batch or read-batch\n");
+	    return 0;
+	}
+
         *argv = poptGetArgs(pc);
         if (*argv)
                 *argc = count_args(*argv);
@@ -561,8 +581,8 @@ void server_options(char **args,int *argc)
 	static char mdelete[30];
 	static char mwindow[30];
 	static char bw[50];
-	static char fext[20];
-	static char wbatch[14];
+	/* Leave room for ``--(write|read)-batch='' */
+	static char fext[MAXPATHLEN + 15];
 
 	int i, x;
 
@@ -644,13 +664,14 @@ void server_options(char **args,int *argc)
 		args[ac++] = mdelete;
 	}    
 	
-	if (write_batch) {
-		snprintf(wbatch,sizeof(wbatch),"--write-batch");
-		args[ac++] = wbatch;
-	}
-
-	if (batch_ext != NULL) {
-		snprintf(fext,sizeof(fext),"--read-batch=%s",batch_ext);
+	if (batch_prefix != NULL) {
+		char *fmt = "";
+		if (write_batch)
+		    fmt = "--write-batch=%s";
+		else
+		if (read_batch)
+		    fmt = "--read-batch=%s";
+		snprintf(fext,sizeof(fext),fmt,batch_prefix);
 		args[ac++] = fext;
 	}
 

rsync.yo

@@ -280,8 +280,8 @@ verb(
      --log-format=FORMAT     log file transfers using specified format
      --password-file=FILE    get password from FILE
      --bwlimit=KBPS          limit I/O bandwidth, KBytes per second
-     --read-batch=FILE       read batch file
-     --write-batch           write batch file
+     --read-batch=PREFIX     read batch fileset starting with PREFIX
+     --write-batch=PREFIX    write batch fileset starting with PREFIX
  -h, --help                  show this help screen
 
 
@@ -706,10 +706,13 @@ transfer was too fast, it will wait before sending the next data block. The
 result is an average transfer rate equalling the specified limit. A value
 of zero specifies no limit.
 
-dit(bf(--read-batch)) Apply a previously generated change batch.
+dit(bf(--write-batch=PREFIX)) Generate a set of files that can be
+transferred as a batch update. Each filename in the set starts with
+PREFIX. See the "BATCH MODE" section for details.
 
-dit(bf(--write-batch)) Generate a set of files that can be transferred
-as a batch update.
+dit(bf(--read-batch=PREFIX)) Apply a previously generated change batch,
+using the fileset whose filenames start with PREFIX. See the "BATCH
+MODE" section for details.
 
 enddit()
 
@@ -804,26 +807,83 @@ itemize(
 manpagesection(BATCH MODE)
 
 bf(Note:) Batch mode should be considered experimental in this version
-of rsync.  The interface or behaviour may change before it stabilizes.
+of rsync. The interface or behaviour may change before it stabilizes.
+
+Batch mode can be used to apply the same set of updates to many
+identical systems. Suppose one has a tree which is replicated on a
+number of hosts.  Now suppose some changes have been made to this
+source tree and those changes need to be propagated to the other
+hosts. In order to do this using batch mode, rsync is run with the
+write-batch option to apply the changes made to the source tree to one
+of the destination trees.  The write-batch option causes the rsync
+client to store the information needed to repeat this operation against
+other destination trees in a batch update fileset (see below).  The
+filename of each file in the fileset starts with a prefix specified by
+the user as an argument to the write-batch option.  This fileset is
+then copied to each remote host, where rsync is run with the read-batch
+option, again specifying the same prefix, and the destination tree.
+Rsync updates the destination tree using the information stored in the
+batch update fileset.
+
+The fileset consists of 4 files:
 
-The following call generates 4 files that encapsulate the information
-for synchronizing the contents of bf(target_dir) with the updates found in
-bf(src_dir)
-
-quote(
-$ rsync --write-batch [other rsync options here] \nl()
-           /somewhere/src_dir /somewhere/target_dir
+itemize(
+it() bf(<prefix>.rsync_argvs) command-line arguments
+it() bf(<prefix>.rsync_flist) rsync internal file metadata
+it() bf(<prefix>.rsync_csums) rsync checksums
+it() bf(<prefix>.rsync_delta) data blocks for file update & change
 )
 
-The generated files are labeled with a common timestamp:
+The .rsync_argvs file contains a command-line suitable for updating a
+destination tree using that batch update fileset. It can be executed
+using a Bourne(-like) shell, optionally passing in an alternate
+destination tree pathname which is then used instead of the original
+path. This is useful when the destination tree path differs from the
+original destination tree path.
 
-itemize(
-it() bf(rsync_argvs.<timestamp>) command-line arguments
-it() bf(rsync_flist.<timestamp>) rsync internal file metadata
-it() bf(rsync_csums.<timestamp>) rsync checksums
-it() bf(rsync_delta.<timestamp>) data blocks for file update & change
+Generating the batch update fileset once saves having to perform the
+file status, checksum and data block generation more than once when
+updating multiple destination trees. Multicast transport protocols can
+be used to transfer the batch update files in parallel to many hosts at
+once, instead of sending the same data to every host individually.
+
+Example:
+
+verb(
+$ rsync --write_batch=pfx -a /source/dir/ /adest/dir/
+$ rcp pfx.rsync_* remote:
+$ rsh remote rsync --read_batch=pfx -a /bdest/dir/
+# or alternatively
+$ rsh remote ./pfx.rsync_argvs /bdest/dir/
 )
 
+In this example, rsync is used to update /adest/dir/ with /source/dir/
+and the information to repeat this operation is stored in the files
+pfx.rsync_*. These files are then copied to the machine named "remote".
+Rsync is then invoked on "remote" to update /bdest/dir/ the same way as
+/adest/dir/. The last line shows the rsync_argvs file being used to
+invoke rsync.
+
+Caveats:
+
+The read-batch option expects the destination tree it is meant to update
+to be identical to the destination tree that was used to create the
+batch update fileset.  When a difference between the destination trees
+is encountered the update will fail at that point, leaving the
+destination tree in a partially updated state. In that case, rsync can
+be used in its regular (non-batch) mode of operation to fix up the
+destination tree.
+
+The rsync version used on all destinations should be identical to the
+one used on the original destination.
+
+The -z/--compress option does not work in batch mode and yields a usage
+error. A separate compression tool can be used instead to reduce the
+size of the batch update files for transport to the destination.
+
+The -n/--dryrun option does not work in batch mode and yields a runtime
+error.
+
 See bf(http://www.ils.unc.edu/i2dsi/unc_rsync+.html) for papers and technical
 reports.