Updated indexing for accurate progress and improved raw ndx values.

The sending side now has a sorted file-list in iconv mode so that it
can output progress in sorted order.  Simplified the over-the-wire
index values to ensure both sides will always agree on the values.
Optimized the allocation of the dir_flist->sorted array on the
receiving side with --iconv and incremental recursion.

flist.c

@@ -121,9 +121,8 @@ static char tmp_sum[MAX_DIGEST_LEN];
 static char empty_sum[MAX_DIGEST_LEN];
 static int flist_count_offset; /* for --delete --progress */
 static int dir_count = 0;
-static int high_hlink_ndx;
 
-static void clean_flist(struct file_list *flist, int strip_root);
+static void flist_sort_and_clean(struct file_list *flist, int strip_root);
 static void output_flist(struct file_list *flist);
 
 void init_flist(void)
@@ -457,8 +456,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_
 			struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino);
 			first_hlink_ndx = (int32)(long)np->data - 1;
 			if (first_hlink_ndx < 0) {
-				high_hlink_ndx = ndx + first_ndx;
-				np->data = (void*)(long)(high_hlink_ndx + 1);
+				np->data = (void*)(long)(first_ndx + ndx + 1);
 				xflags |= XMIT_HLINK_FIRST;
 			}
 			xflags |= XMIT_HLINKED;
@@ -939,8 +937,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 	if (preserve_hard_links && xflags & XMIT_HLINKED) {
 		if (protocol_version >= 30) {
 			if (xflags & XMIT_HLINK_FIRST) {
-				high_hlink_ndx = flist->ndx_start + flist->used;
-				F_HL_GNUM(file) = high_hlink_ndx;
+				F_HL_GNUM(file) = flist->ndx_start + flist->used;
 			} else
 				F_HL_GNUM(file) = first_hlink_ndx;
 		} else {
@@ -1719,11 +1716,7 @@ void send_extra_file_list(int f, int at_least)
 		} else
 			flist->sorted = flist->files;
 
-		clean_flist(flist, 0);
-
-		flist->ndx_end = flist->ndx_start + flist->used - 1;
-		if (!need_unsorted_flist)
-			flist->ndx_end -= (dir_count - dstart);
+		flist_sort_and_clean(flist, 0);
 
 		add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart);
 		flist_done_allocating(flist);
@@ -2021,27 +2014,15 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	 * recursion mode, the sender marks duplicate dirs so that it can
 	 * send them together in a single file-list. */
 	if (need_unsorted_flist) {
-		if (inc_recurse) {
-			if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-				out_of_memory("send_file_list");
-			memcpy(flist->sorted, flist->files,
-			       flist->used * sizeof (struct file_struct*));
-			clean_flist(flist, 0);
-		} else {
-			flist->sorted = flist->files;
-			flist->low = 0;
-			flist->high = flist->used - 1;
-		}
-	} else {
+		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
+			out_of_memory("send_file_list");
+		memcpy(flist->sorted, flist->files,
+		       flist->used * sizeof (struct file_struct*));
+	} else
 		flist->sorted = flist->files;
-		clean_flist(flist, 0);
-	}
+	flist_sort_and_clean(flist, 0);
 	file_total += flist->used;
 
-	/* We don't subtract dir_count for the first send since we
-	 * might have one or more dot dirs which need to get sent. */
-	flist->ndx_end = flist->ndx_start + flist->used - 1;
-
 	if (!numeric_ids && !inc_recurse)
 		send_id_list(f);
 
@@ -2140,10 +2121,6 @@ struct file_list *recv_file_list(int f)
 	}
 	file_total += flist->used;
 
-	flist->ndx_end = flist->ndx_start + flist->used - 1;
-	if (inc_recurse && !need_unsorted_flist && flist->ndx_start > 1)
-		flist->ndx_end -= dir_flist->used - dstart;
-
 	if (verbose > 2)
 		rprintf(FINFO, "received %d names\n", flist->used);
 
@@ -2161,9 +2138,13 @@ struct file_list *recv_file_list(int f)
 		memcpy(flist->sorted, flist->files,
 		       flist->used * sizeof (struct file_struct*));
 		if (inc_recurse && dir_flist->used > dstart) {
-			dir_flist->sorted = realloc_array(dir_flist->sorted,
-						struct file_struct *,
-						dir_flist->used);
+			static int dir_flist_malloced = 0;
+			if (dir_flist_malloced < dir_flist->malloced) {
+				dir_flist->sorted = realloc_array(dir_flist->sorted,
+							struct file_struct *,
+							dir_flist->malloced);
+				dir_flist_malloced = dir_flist->malloced;
+			}
 			memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
 			       (dir_flist->used - dstart) * sizeof (struct file_struct*));
 			fsort(dir_flist->sorted + dstart, dir_flist->used - dstart);
@@ -2181,7 +2162,7 @@ struct file_list *recv_file_list(int f)
 	else if (f >= 0)
 		recv_id_list(f, flist);
 
-	clean_flist(flist, relative_paths);
+	flist_sort_and_clean(flist, relative_paths);
 
 	if (protocol_version < 30) {
 		/* Recv the io_error flag */
@@ -2321,18 +2302,19 @@ struct file_list *flist_new(int flags, char *msg)
 			if (!flist->file_pool)
 				out_of_memory(msg);
 
-			flist->ndx_start = inc_recurse ? 1 : 0;
+			flist->ndx_start = flist->flist_num = inc_recurse ? 1 : 0;
 
 			first_flist = cur_flist = flist->prev = flist;
 		} else {
+			struct file_list *prev = first_flist->prev;
+
 			flist->file_pool = first_flist->file_pool;
 
-			flist->ndx_start = first_flist->prev->ndx_end + 2;
-			if (flist->ndx_start <= high_hlink_ndx)
-				flist->ndx_start = high_hlink_ndx + 1;
+			flist->ndx_start = prev->ndx_start + prev->used + 1;
+			flist->flist_num = prev->flist_num + 1;
 
-			flist->prev = first_flist->prev;
-			flist->prev->next = first_flist->prev = flist;
+			flist->prev = prev;
+			prev->next = first_flist->prev = flist;
 		}
 		flist->pool_boundary = pool_boundary(flist->file_pool, 0);
 		flist_cnt++;
@@ -2378,7 +2360,7 @@ void flist_free(struct file_list *flist)
 
 /* This routine ensures we don't have any duplicate names in our file list.
  * duplicate names can cause corruption because of the pipelining. */
-static void clean_flist(struct file_list *flist, int strip_root)
+static void flist_sort_and_clean(struct file_list *flist, int strip_root)
 {
 	char fbuf[MAXPATHLEN];
 	int i, prev_i;
@@ -2558,8 +2540,8 @@ static void output_flist(struct file_list *flist)
 	const char *who = who_am_i();
 	int i;
 
-	rprintf(FINFO, "[%s] flist start=%d, end=%d, used=%d, low=%d, high=%d\n",
-		who, flist->ndx_start, flist->ndx_end, flist->used, flist->low, flist->high);
+	rprintf(FINFO, "[%s] flist start=%d, used=%d, low=%d, high=%d\n",
+		who, flist->ndx_start, flist->used, flist->low, flist->high);
 	for (i = 0; i < flist->used; i++) {
 		file = flist->files[i];
 		if ((am_root || am_sender) && uid_ndx) {
@@ -2809,7 +2791,7 @@ struct file_list *get_dirlist(char *dirname, int dlen, int ignore_filter_rules)
 
 	prune_empty_dirs = 0;
 	dirlist->sorted = dirlist->files;
-	clean_flist(dirlist, 0);
+	flist_sort_and_clean(dirlist, 0);
 	prune_empty_dirs = save_prune_empty_dirs;
 
 	if (verbose > 3)

progress.c

@@ -22,8 +22,10 @@
 
 #include "rsync.h"
 
-extern struct stats stats;
 extern int am_server;
+extern int need_unsorted_flist;
+extern struct stats stats;
+extern struct file_list *cur_flist;
 
 #define PROGRESS_HISTORY_SECS 5
 
@@ -41,6 +43,7 @@ struct progress_history {
 static struct progress_history ph_start;
 static struct progress_history ph_list[PROGRESS_HISTORY_SECS];
 static int newest_hpos, oldest_hpos;
+static int current_file_index;
 
 static unsigned long msdiff(struct timeval *t1, struct timeval *t2)
 {
@@ -104,7 +107,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now,
 	if (is_last) {
 		snprintf(eol, sizeof eol, " (xfer#%d, to-check=%d/%d)\n",
 			stats.num_transferred_files,
-			stats.num_files - stats.current_file_index - 1,
+			stats.num_files - current_file_index - 1,
 			stats.num_files);
 	} else
 		strlcpy(eol, "\r", sizeof eol);
@@ -112,6 +115,15 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now,
 		human_num(ofs), pct, rate, units, rembuf, eol);
 }
 
+void set_current_file_index(struct file_struct *file, int ndx)
+{
+	if (need_unsorted_flist)
+		current_file_index = flist_find(cur_flist, file) + cur_flist->ndx_start;
+	else
+		current_file_index = ndx;
+	current_file_index -= cur_flist->flist_num;
+}
+
 void end_progress(OFF_T size)
 {
 	if (!am_server) {

receiver.c

@@ -481,9 +481,11 @@ int recv_files(int f_in, char *local_name)
 			}
 		}
 
-		stats.current_file_index = ndx;
+		if (!am_server && do_progress)
+			set_current_file_index(file, ndx);
 		stats.num_transferred_files++;
 		stats.total_transferred_size += F_LENGTH(file);
+
 		cleanup_got_literal = 0;
 
 		if (server_filter_list.head

rsync.c

@@ -264,12 +264,13 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 	}
 
 	if (!(flist = flist_for_ndx(ndx))) {
+		int start, used;
 	  invalid_ndx:
+		start = first_flist ? first_flist->ndx_start : 0;
+		used = first_flist ? first_flist->used : 0;
 		rprintf(FERROR,
 			"Invalid file index: %d (%d - %d) with iflags %x [%s]\n",
-			ndx, first_flist ? first_flist->ndx_start - 1 : -1,
-			first_flist ? first_flist->prev->ndx_end : -1,
-			iflags, who_am_i());
+			ndx, start - 1, start + used -1, iflags, who_am_i());
 		exit_cleanup(RERR_PROTOCOL);
 	}
 	cur_flist = flist;
@@ -572,7 +573,7 @@ struct file_list *flist_for_ndx(int ndx)
 			return NULL;
 		flist = flist->prev;
 	}
-	while (ndx > flist->ndx_end) {
+	while (ndx >= flist->ndx_start + flist->used) {
 		if (!(flist = flist->next))
 			return NULL;
 	}

rsync.h

@@ -93,7 +93,7 @@
 /* This is used when working on a new protocol version in CVS, and should
  * be a new non-zero value for each CVS change that affects the protocol.
  * It must ALWAYS be 0 when the protocol goes final! */
-#define SUBPROTOCOL_VERSION 15
+#define SUBPROTOCOL_VERSION 16
 
 /* We refuse to interoperate with versions that are not in this range.
  * Note that we assume we'll work with later versions: the onus is on
@@ -723,7 +723,7 @@ struct file_list {
 	int used, malloced;
 	int low, high;  /* 0-relative index values excluding empties */
 	int ndx_start;  /* the start offset for inc_recurse mode */
-	int ndx_end;    /* the end offset for inc_recurse mode */
+	int flist_num;  /* 1-relative file_list number or 0 */
 	int parent_ndx; /* dir_flist index of parent directory */
 	int in_progress, to_redo;
 };
@@ -814,7 +814,6 @@ struct stats {
 	int64 flist_size;
 	int num_files;
 	int num_transferred_files;
-	int current_file_index;
 };
 
 struct chmod_mode_struct;

sender.c

@@ -265,7 +265,8 @@ void send_files(int f_in, int f_out)
 		updating_basis_file = inplace && (protocol_version >= 29
 			? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0);
 
-		stats.current_file_index = ndx;
+		if (!am_server && do_progress)
+			set_current_file_index(file, ndx);
 		stats.num_transferred_files++;
 		stats.total_transferred_size += F_LENGTH(file);