Support new --append option.

generator.c

@@ -54,6 +54,7 @@ extern int delay_updates;
 extern int update_only;
 extern int opt_ignore_existing;
 extern int inplace;
+extern int append_mode;
 extern int make_backups;
 extern int csum_length;
 extern int ignore_times;
@@ -470,35 +471,42 @@ static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 	OFF_T offset = 0;
 
 	sum_sizes_sqroot(&sum, len);
+	write_sum_head(f_out, &sum);
+
+	if (append_mode > 0 && f_copy < 0)
+		return;
 
 	if (len > 0)
 		mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
 	else
 		mapbuf = NULL;
 
-	write_sum_head(f_out, &sum);
-
 	for (i = 0; i < sum.count; i++) {
 		int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
 		char *map = map_ptr(mapbuf, offset, n1);
-		uint32 sum1 = get_checksum1(map, n1);
 		char sum2[SUM_LENGTH];
+		uint32 sum1;
+
+		len -= n1;
+		offset += n1;
 
-		if (f_copy >= 0)
+		if (f_copy >= 0) {
 			full_write(f_copy, map, n1);
+			if (append_mode > 0)
+				continue;
+		}
 
+		sum1 = get_checksum1(map, n1);
 		get_checksum2(map, n1, sum2);
 
 		if (verbose > 3) {
 			rprintf(FINFO,
 				"chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
-				(double)i, (double)offset, (long)n1,
+				(double)i, (double)offset - n1, (long)n1,
 				(unsigned long)sum1);
 		}
 		write_int(f_out, sum1);
 		write_buf(f_out, sum2, sum.s2length);
-		len -= n1;
-		offset += n1;
 	}
 
 	if (mapbuf)
@@ -1007,6 +1015,9 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		return;
 	}
 
+	if (append_mode && st.st_size > file->length)
+		return;
+
 	if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
 		;
 	else if (fnamecmp_type == FNAMECMP_FUZZY)
@@ -1180,7 +1191,7 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
 		do_delete_pass(flist);
 	do_progress = 0;
 
-	if (whole_file < 0)
+	if (append_mode || whole_file < 0)
 		whole_file = 0;
 	if (verbose >= 2) {
 		rprintf(FINFO, "delta-transmission %s\n",
@@ -1239,6 +1250,8 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
 	only_existing = max_size = opt_ignore_existing = 0;
 	update_only = always_checksum = size_only = 0;
 	ignore_times = 1;
+	if (append_mode)  /* resend w/o append mode */
+		append_mode = -1; /* ... but only longer files */
 	make_backups = 0; /* avoid a duplicate backup for inplace processing */
 
 	if (verbose > 2)

match.c

@@ -23,6 +23,7 @@ extern int verbose;
 extern int am_server;
 extern int do_progress;
 extern int checksum_seed;
+extern int append_mode;
 
 int updating_basis_file;
 
@@ -330,6 +331,21 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
 
 	sum_init(checksum_seed);
 
+	if (append_mode) {
+		OFF_T j = 0;
+		for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
+			sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
+				   CHUNK_SIZE);
+			last_match = j;
+		}
+		if (last_match < s->flength) {
+			int32 len = s->flength - last_match;
+			sum_update(map_ptr(buf, last_match, len), len);
+			last_match = s->flength;
+		}
+		s->count = 0;
+	}
+
 	if (len > 0 && s->count > 0) {
 		build_hash_table(s);
 
@@ -343,7 +359,7 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
 	} else {
 		OFF_T j;
 		/* by doing this in pieces we avoid too many seeks */
-		for (j = CHUNK_SIZE; j < len; j += CHUNK_SIZE)
+		for (j = last_match + CHUNK_SIZE; j < len; j += CHUNK_SIZE)
 			matched(f, s, buf, j, -2);
 		matched(f, s, buf, len, -1);
 	}

receiver.c

@@ -45,6 +45,7 @@ extern int remove_sent_files;
 extern int module_id;
 extern int ignore_errors;
 extern int orig_umask;
+extern int append_mode;
 extern int keep_partial;
 extern int checksum_seed;
 extern int inplace;
@@ -212,6 +213,28 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 
 	sum_init(checksum_seed);
 
+	if (append_mode) {
+		OFF_T j;
+		sum.flength = (OFF_T)sum.count * sum.blength;
+		if (sum.remainder)
+			sum.flength -= sum.blength - sum.remainder;
+		for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
+			sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
+				   CHUNK_SIZE);
+			offset = j;
+		}
+		if (offset < sum.flength) {
+			int32 len = sum.flength - offset;
+			sum_update(map_ptr(mapbuf, offset, len), len);
+			offset = sum.flength;
+		}
+		if (fd != -1 && do_lseek(fd, offset, SEEK_SET) != offset) {
+			rsyserr(FERROR, errno, "lseek failed on %s",
+				full_fname(fname));
+			exit_cleanup(RERR_FILEIO);
+		}
+	}
+
 	while ((i = recv_token(f_in, &data)) != 0) {
 		if (do_progress)
 			show_progress(offset, total_size);
@@ -417,6 +440,7 @@ int recv_files(int f_in, struct file_list *flist, char *local_name)
 			send_msg(MSG_DONE, "", 0);
 			if (keep_partial && !partial_dir)
 				make_backups = 0; /* prevents double backup */
+			append_mode = 0;
 			continue;
 		}
 

sender.c

@@ -27,6 +27,7 @@ extern int log_before_transfer;
 extern int log_format_has_i;
 extern int daemon_log_format_has_i;
 extern int csum_length;
+extern int append_mode;
 extern int io_error;
 extern int allowed_lull;
 extern int protocol_version;
@@ -72,6 +73,13 @@ static struct sum_struct *receive_sums(int f)
 			(double)s->count, (long)s->blength, (long)s->remainder);
 	}
 
+	if (append_mode) {
+		s->flength = (OFF_T)s->count * s->blength;
+		if (s->remainder)
+			s->flength -= s->blength - s->remainder;
+		return s;
+	}
+
 	if (s->count == 0)
 		return(s);
 
@@ -231,6 +239,7 @@ void send_files(struct file_list *flist, int f_out, int f_in)
 			/* For inplace: redo phase turns off the backup
 			 * flag so that we do a regular inplace send. */
 			make_backups = 0;
+			append_mode = 0;
 			continue;
 		}