qfile-agent: move data handling code to libqubes-rpc-filecopy

This makes the qfile packing code reusable, for example for some dom0
tool. Now qfile-agent.c is only an interface for underlying library.

QubesOS/qubes-issues#1324

qubes-rpc/Makefile

@@ -5,11 +5,11 @@ vm-file-editor: vm-file-editor.o
 	$(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy
 qopen-in-vm: qopen-in-vm.o gui-fatal.o
 	$(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy
-qfile-agent: qfile-agent.o gui-fatal.o qfile-utils.o
+qfile-agent: qfile-agent.o gui-fatal.o
 	$(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy
 qfile-unpacker: qfile-unpacker.o gui-fatal.o
 	$(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy
-tar2qfile: qfile-utils.o tar2qfile.o gui-fatal.o
+tar2qfile: tar2qfile.o gui-fatal.o
 	$(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy
 
 clean:

qubes-rpc/gui-fatal.c

@@ -53,6 +53,11 @@ void gui_fatal(const char *fmt, ...)
 	exit(1);
 }
 
+void qfile_gui_fatal(const char *fmt, va_list args) {
+	produce_message("Fatal error", fmt, args);
+    exit(1);
+}
+
 void gui_nonfatal(const char *fmt, ...)
 {
 	va_list args;

qubes-rpc/gui-fatal.h

@@ -1,2 +1,3 @@
 void gui_fatal(const char *fmt, ...);
 void gui_nonfatal(const char *fmt, ...);
+void qfile_gui_fatal(const char *fmt, va_list args);

qubes-rpc/qfile-agent.c

@@ -1,4 +1,67 @@
-#include "qfile-utils.h"
+#define _GNU_SOURCE
+#include <dirent.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <malloc.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <gui-fatal.h>
+#include <libqubes-rpc-filecopy.h>
+
+enum {
+	PROGRESS_FLAG_NORMAL,
+	PROGRESS_FLAG_INIT,
+	PROGRESS_FLAG_DONE
+};
+
+void do_notify_progress(long long total, int flag)
+{
+	const char *du_size_env = getenv("FILECOPY_TOTAL_SIZE");
+	const char *progress_type_env = getenv("PROGRESS_TYPE");
+	const char *saved_stdout_env = getenv("SAVED_FD_1");
+	int ignore;
+	if (!progress_type_env)
+		return;
+	if (!strcmp(progress_type_env, "console") && du_size_env) {
+		char msg[256];
+		snprintf(msg, sizeof(msg), "sent %lld/%lld KB\r",
+			 total / 1024, strtoull(du_size_env, NULL, 0));
+		ignore = write(2, msg, strlen(msg));
+		if (flag == PROGRESS_FLAG_DONE)
+			ignore = write(2, "\n", 1);
+	}
+	if (!strcmp(progress_type_env, "gui") && saved_stdout_env) {
+		char msg[256];
+		snprintf(msg, sizeof(msg), "%lld\n", total);
+		ignore = write(strtoul(saved_stdout_env, NULL, 0), msg,
+				strlen(msg));
+	}
+	if (ignore < 0) {
+		/* silence gcc warning */
+	}
+}
+
+void notify_progress(int size, int flag)
+{
+	static long long total = 0;
+	static long long prev_total = 0;
+	total += size;
+	if (total > prev_total + PROGRESS_NOTIFY_DELTA
+	    || (flag != PROGRESS_FLAG_NORMAL)) {
+		// check for possible error from qfile-unpacker; if error occured,
+		// exit() will be called, so don't bother with current state
+		// (notify_progress can be called as callback from copy_file())
+		if (flag == PROGRESS_FLAG_NORMAL)
+			wait_for_result();
+		do_notify_progress(total, flag);
+		prev_total = total;
+	}
+}
+
 
 char *get_abs_path(const char *cwd, const char *pathname)
 {
@@ -11,53 +74,18 @@ char *get_abs_path(const char *cwd, const char *pathname)
 		return ret;
 }
 
-int do_fs_walk(const char *file)
-{
-	char *newfile;
-	struct stat st;
-	struct dirent *ent;
-	DIR *dir;
-
-	if (lstat(file, &st))
-		gui_fatal("stat %s", file);
-	single_file_processor(file, &st);
-	if (!S_ISDIR(st.st_mode))
-		return 0;
-	dir = opendir(file);
-	if (!dir)
-		gui_fatal("opendir %s", file);
-	while ((ent = readdir(dir))) {
-		char *fname = ent->d_name;
-		if (!strcmp(fname, ".") || !strcmp(fname, ".."))
-			continue;
-		if (asprintf(&newfile, "%s/%s", file, fname) >= 0) {
-			do_fs_walk(newfile);
-			free(newfile);
-		} else {
-			fprintf(stderr, "asprintf failed\n");
-			exit(1);
-		}
-	}
-	closedir(dir);
-	// directory metadata is resent; this makes the code simple,
-	// and the atime/mtime is set correctly at the second time
-	single_file_processor(file, &st);
-	return 0;
-}
-
 int main(int argc, char **argv)
 {
 	int i;
 	char *entry;
 	char *cwd;
 	char *sep;
+	int ignore_symlinks = 0;
 
-	signal(SIGPIPE, SIG_IGN);
-	// this will allow checking for possible feedback packet in the middle of transfer
-	set_nonblock(0);
+	qfile_pack_init();
+	register_error_handler(qfile_gui_fatal);
 	register_notify_progress(&notify_progress);
 	notify_progress(0, PROGRESS_FLAG_INIT);
-	crc32_sum = 0;
 	cwd = getcwd(NULL, 0);
 	for (i = 1; i < argc; i++) {
 		if (strcmp(argv[i], "--ignore-symlinks")==0) {
@@ -80,7 +108,7 @@ int main(int argc, char **argv)
 			}
 		} else if (chdir(entry))
 			gui_fatal("chdir to %s", entry);
-		do_fs_walk(sep + 1);
+		do_fs_walk(sep + 1, ignore_symlinks);
 		free(entry);
 	}
 	notify_end_and_wait_for_result();

qubes-rpc/qfile-utils.c

@@ -1,197 +0,0 @@
-
-#include <qfile-utils.h>
-
-unsigned long crc32_sum;
-int ignore_symlinks = 0;
-int ignore_quota_error = 0;
-
-void notify_progress(int size, int flag)
-{
-	static long long total = 0;
-	static long long prev_total = 0;
-	total += size;
-	if (total > prev_total + PROGRESS_NOTIFY_DELTA
-	    || (flag != PROGRESS_FLAG_NORMAL)) {
-		// check for possible error from qfile-unpacker; if error occured,
-		// exit() will be called, so don't bother with current state
-		// (notify_progress can be called as callback from copy_file())
-		if (flag == PROGRESS_FLAG_NORMAL)
-			wait_for_result();
-		do_notify_progress(total, flag);
-		prev_total = total;
-	}
-}
-
-void do_notify_progress(long long total, int flag)
-{
-	const char *du_size_env = getenv("FILECOPY_TOTAL_SIZE");
-	const char *progress_type_env = getenv("PROGRESS_TYPE");
-	const char *saved_stdout_env = getenv("SAVED_FD_1");
-	int ignore;
-	if (!progress_type_env)
-		return;
-	if (!strcmp(progress_type_env, "console") && du_size_env) {
-		char msg[256];
-		snprintf(msg, sizeof(msg), "sent %lld/%lld KB\r",
-			 total / 1024, strtoull(du_size_env, NULL, 0));
-		ignore = write(2, msg, strlen(msg));
-		if (flag == PROGRESS_FLAG_DONE)
-			ignore = write(2, "\n", 1);
-	}
-	if (!strcmp(progress_type_env, "gui") && saved_stdout_env) {
-		char msg[256];
-		snprintf(msg, sizeof(msg), "%lld\n", total);
-		ignore = write(strtoul(saved_stdout_env, NULL, 0), msg,
-				strlen(msg));
-	}
-	if (ignore < 0) {
-		/* silence gcc warning */
-	}
-}
-
-void notify_end_and_wait_for_result(void)
-{
-	struct file_header end_hdr;
-
-	/* nofity end of transfer */
-	memset(&end_hdr, 0, sizeof(end_hdr));
-	end_hdr.namelen = 0;
-	end_hdr.filelen = 0;
-	write_all_with_crc(1, &end_hdr, sizeof(end_hdr));
-
-	set_block(0);
-	wait_for_result();
-}
-
-int write_all_with_crc(int fd, const void *buf, int size)
-{
-	crc32_sum = Crc32_ComputeBuf(crc32_sum, buf, size);
-	return write_all(fd, buf, size);
-}
-
-void wait_for_result(void)
-{
-	struct result_header hdr;
-	struct result_header_ext hdr_ext;
-	char last_filename[MAX_PATH_LENGTH + 1];
-	char last_filename_prefix[] = "; Last file: ";
-
-	if (!read_all(0, &hdr, sizeof(hdr))) {
-		if (errno == EAGAIN) {
-			// no result sent and stdin still open
-			return;
-		} else {
-			// other read error or EOF
-			exit(1);	// hopefully remote has produced error message
-		}
-	}
-	if (!read_all(0, &hdr_ext, sizeof(hdr_ext))) {
-		// remote used old result_header struct
-		hdr_ext.last_namelen = 0;
-	}
-	if (hdr_ext.last_namelen > MAX_PATH_LENGTH) {
-		// read only at most MAX_PATH_LENGTH chars
-		hdr_ext.last_namelen = MAX_PATH_LENGTH;
-	}
-	if (!read_all(0, last_filename, hdr_ext.last_namelen)) {
-		fprintf(stderr, "Failed to get last filename\n");
-		hdr_ext.last_namelen = 0;
-	}
-	last_filename[hdr_ext.last_namelen] = '\0';
-	if (!hdr_ext.last_namelen)
-		/* set prefix to empty string */
-		last_filename_prefix[0] = '\0';
-
-	errno = hdr.error_code;
-	if (hdr.error_code != 0) {
-		switch (hdr.error_code) {
-			case EEXIST:
-				gui_fatal("File copy: not overwriting existing file. Clean QubesIncoming dir, and retry copy%s%s", last_filename_prefix, last_filename);
-				break;
-			case EINVAL:
-				gui_fatal("File copy: Corrupted data from packer%s%s", last_filename_prefix, last_filename);
-				break;
-			case EDQUOT:
-				if (ignore_quota_error) {
-					/* skip also CRC check as sender and receiver might be
-					 * desynchronized in this case */
-					return;
-				}
-				/* fall though */
-			default:
-				gui_fatal("File copy: %s%s%s",
-						strerror(hdr.error_code), last_filename_prefix, last_filename);
-		}
-	}
-	if (hdr.crc32 != crc32_sum) {
-		gui_fatal("File transfer failed: checksum mismatch");
-	}
-}
-
-void write_headers(const struct file_header *hdr, const char *filename)
-{
-	if (!write_all_with_crc(1, hdr, sizeof(*hdr))
-	    || !write_all_with_crc(1, filename, hdr->namelen)) {
-		set_block(0);
-		wait_for_result();
-		exit(1);
-	}
-}
-
-int single_file_processor(const char *filename, const struct stat *st)
-{
-	struct file_header hdr;
-	int fd;
-	mode_t mode = st->st_mode;
-
-	hdr.namelen = strlen(filename) + 1;
-	hdr.mode = mode;
-	hdr.atime = st->st_atim.tv_sec;
-	hdr.atime_nsec = st->st_atim.tv_nsec;
-	hdr.mtime = st->st_mtim.tv_sec;
-	hdr.mtime_nsec = st->st_mtim.tv_nsec;
-
-	if (S_ISREG(mode)) {
-		int ret;
-		fd = open(filename, O_RDONLY);
-		if (fd < 0)
-			gui_fatal("open %s", filename);
-		hdr.filelen = st->st_size;
-		write_headers(&hdr, filename);
-		ret = copy_file(1, fd, hdr.filelen, &crc32_sum);
-		if (ret != COPY_FILE_OK) {
-			if (ret != COPY_FILE_WRITE_ERROR)
-				gui_fatal("Copying file %s: %s", filename,
-					  copy_file_status_to_str(ret));
-			else {
-				set_block(0);
-				wait_for_result();
-				exit(1);
-			}
-		}
-		close(fd);
-	}
-	if (S_ISDIR(mode)) {
-		hdr.filelen = 0;
-		write_headers(&hdr, filename);
-	}
-	if (S_ISLNK(mode) && !ignore_symlinks) {
-		char name[st->st_size + 1];
-		if (readlink(filename, name, sizeof(name)) != st->st_size)
-			gui_fatal("readlink %s", filename);
-		hdr.filelen = st->st_size;
-		write_headers(&hdr, filename);
-		if (!write_all_with_crc(1, name, st->st_size)) {
-			set_block(0);
-			wait_for_result();
-			exit(1);
-		}
-	}
-	// check for possible error from qfile-unpacker
-	wait_for_result();
-	return 0;
-}
-
-
-
-

qubes-rpc/qfile-utils.h

@@ -1,40 +0,0 @@
-
-#ifndef _LIBQUBES_QFILE_UTILS_H
-#define _LIBQUBES_QFILE_UTILS_H 1
-
-#define _GNU_SOURCE
-#include <dirent.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <signal.h>
-#include <fcntl.h>
-#include <malloc.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <gui-fatal.h>
-#include <libqubes-rpc-filecopy.h>
-
-enum {
-	PROGRESS_FLAG_NORMAL,
-	PROGRESS_FLAG_INIT,
-	PROGRESS_FLAG_DONE
-};
-
-extern unsigned long crc32_sum;
-extern int ignore_symlinks;
-
-void notify_progress(int size, int flag);
-void do_notify_progress(long long total, int flag);
-void notify_end_and_wait_for_result(void);
-
-void write_headers(const struct file_header *hdr, const char *filename);
-
-int write_all_with_crc(int fd, const void *buf, int size);
-
-int single_file_processor(const char *filename, const struct stat *st);
-
-void wait_for_result(void);
-
-#endif /* _LIBQUBES_QFILE_UTILS_H */

qubes-rpc/tar2qfile.c

@@ -46,7 +46,7 @@
 #include <stdio.h>
 #include <libqubes-rpc-filecopy.h>
 #include <string.h>
-#include <qfile-utils.h>
+#include <gui-fatal.h>
 
 // #define DEBUG
 
@@ -800,7 +800,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s
 #ifdef DEBUG
 	fprintf(stderr,"Writing file content\n");
 #endif
-	ret = copy_file(1, fd, untrusted_hdr->filelen, &crc32_sum);
+	ret = copy_file_with_crc(1, fd, untrusted_hdr->filelen);
 
 #ifdef DEBUG
 	fprintf(stderr,"Copyfile returned with error %d\n",ret);
@@ -960,23 +960,13 @@ int main(int argc, char **argv)
 	int use_stdin = 1;
 	struct filters filters;
 
-	signal(SIGPIPE, SIG_IGN);
-	// this will allow checking for possible feedback packet in the middle of transfer
-	// if disabled, the copy_file process could hang
-	register_notify_progress(&notify_progress);
-	notify_progress(0, PROGRESS_FLAG_INIT);
-	//set_size_limit(1500000000, 2048);
-
-	crc32_sum = 0;
+	qfile_pack_init();
 	/* when extracting backup header, dom0 will terminate the transfer with
 	 * EDQUOT just after getting qubes.xml */
-	ignore_quota_error = 1;
+	set_ignore_quota_error(1);
 	for (i = 1; i < argc; i++) {
 		set_nonblock(0);
-		if (strcmp(argv[i], "--ignore-symlinks")==0) {
-			ignore_symlinks = 1;
-			continue;
-		} else if (strcmp(argv[i], "-")==0) {
+		if (strcmp(argv[i], "-")==0) {
 			use_stdin = 1;
 			i++;
 			break;
@@ -1021,7 +1011,6 @@ int main(int argc, char **argv)
 
 
 	notify_end_and_wait_for_result();
-	notify_progress(0, PROGRESS_FLAG_DONE);
 	return 0;
 }