From f372062c4b0e079253c311571c8cafe7da6ab110 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 7 Aug 2013 11:20:33 +0200 Subject: [PATCH 01/32] makefile: Use the sbindir variable instead of a fixed path --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 52477a4..4ca0c5e 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ VERSION := $(shell cat version) DIST ?= fc18 KDESERVICEDIR ?= /usr/share/kde4/services +SBINDIR ?= /usr/sbin help: @echo "make rpms -- generate binary rpm packages" @@ -54,7 +55,7 @@ install-vm: install -D -m 0440 misc/qubes.sudoers $(DESTDIR)/etc/sudoers.d/qubes install -D -m 0644 misc/qubes.repo $(DESTDIR)/etc/yum.repos.d/qubes.repo install -D -m 0644 misc/serial.conf $(DESTDIR)/usr/lib/qubes/serial.conf - install -D misc/qubes-serial-login $(DESTDIR)/sbin/qubes-serial-login + install -D misc/qubes-serial-login $(DESTDIR)/$(SBINDIR)/qubes-serial-login install -d $(DESTDIR)/usr/share/glib-2.0/schemas/ install -m 0644 misc/org.gnome.settings-daemon.plugins.updates.gschema.override $(DESTDIR)/usr/share/glib-2.0/schemas/ install -d $(DESTDIR)/usr/lib/yum-plugins/ @@ -107,9 +108,9 @@ install-vm: install -d $(DESTDIR)/etc/yum.conf.d touch $(DESTDIR)/etc/yum.conf.d/qubes-proxy.conf - install -d $(DESTDIR)/usr/sbin - install network/qubes-firewall $(DESTDIR)/usr/sbin/ - install network/qubes-netwatcher $(DESTDIR)/usr/sbin/ + install -d $(DESTDIR)/$(SBINDIR) + install network/qubes-firewall $(DESTDIR)/$(SBINDIR)/ + install network/qubes-netwatcher $(DESTDIR)/$(SBINDIR)/ install -d $(DESTDIR)/usr/bin From 2e946106d40a3f64dcf78748a15b0268d7329397 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 7 Aug 2013 11:21:26 +0200 Subject: [PATCH 02/32] archlinux: avoid installing tools into /sbin or /usr/sbin as required by archlinux --- archlinux/PKGBUILD | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/archlinux/PKGBUILD b/archlinux/PKGBUILD index eae108b..ab9fa17 100644 --- a/archlinux/PKGBUILD +++ b/archlinux/PKGBUILD @@ -6,7 +6,7 @@ # Maintainer: Olivier Medoc pkgname=qubes-vm-core pkgver=`cat version` -pkgrel=12 +pkgrel=13 epoch= pkgdesc="The Qubes core files for installation inside a Qubes VM." arch=("x86_64") @@ -48,6 +48,11 @@ sed 's:/sbin/ethtool:ethtool:g' -i network/* sed 's:/sbin/ip:ip:g' -i network/* sed 's:/bin/grep:grep:g' -i network/* +# Fix for archlinux sbindir +sed 's:/usr/sbin/ntpdate:ntpdate:g' -i qubes-rpc/sync-ntp-clock +sed 's:/usr/sbin/qubes-netwatcher:qubes-netwatcher:g' -i vm-systemd/qubes-netwatcher.service +sed 's:/usr/sbin/qubes-firewall:qubes-firewall:g' -i vm-systemd/qubes-firewall.service + for dir in qubes-rpc qrexec misc; do (cd $dir; make) done @@ -55,10 +60,11 @@ done } package() { + # Note: Archlinux removed use of directory such as /sbin /bin /usr/sbin (https://mailman.archlinux.org/pipermail/arch-dev-public/2012-March/022625.html) + + (cd qrexec; make install DESTDIR=$pkgdir SBINDIR=/usr/bin) - (cd qrexec; make install DESTDIR=$pkgdir) - - make install-vm DESTDIR=$pkgdir DIST=archlinux + make install-vm DESTDIR=$pkgdir SBINDIR=/usr/bin DIST=archlinux # Convert module loading to ARCHLINUX mkdir -p $pkgdir/etc/modules-load.d/ From 509efdd583687d32d57e401047a34267d5339783 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 13 Aug 2013 09:36:35 +0200 Subject: [PATCH 03/32] archlinux: fix systemd scripts to ensure that absolute path are used --- archlinux/PKGBUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archlinux/PKGBUILD b/archlinux/PKGBUILD index ab9fa17..346bab5 100644 --- a/archlinux/PKGBUILD +++ b/archlinux/PKGBUILD @@ -49,9 +49,9 @@ sed 's:/sbin/ip:ip:g' -i network/* sed 's:/bin/grep:grep:g' -i network/* # Fix for archlinux sbindir -sed 's:/usr/sbin/ntpdate:ntpdate:g' -i qubes-rpc/sync-ntp-clock -sed 's:/usr/sbin/qubes-netwatcher:qubes-netwatcher:g' -i vm-systemd/qubes-netwatcher.service -sed 's:/usr/sbin/qubes-firewall:qubes-firewall:g' -i vm-systemd/qubes-firewall.service +sed 's:/usr/sbin/ntpdate:/usr/bin/ntpdate:g' -i qubes-rpc/sync-ntp-clock +sed 's:/usr/sbin/qubes-netwatcher:/usr/bin/qubes-netwatcher:g' -i vm-systemd/qubes-netwatcher.service +sed 's:/usr/sbin/qubes-firewall:/usr/bin/qubes-firewall:g' -i vm-systemd/qubes-firewall.service for dir in qubes-rpc qrexec misc; do (cd $dir; make) From c26d4b4d3035d3d44a080061a989c712c9e31713 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 4 Oct 2013 14:36:00 +0200 Subject: [PATCH 04/32] rpc: implemented new rpc helper tool --- qubes-rpc/tar2qfile.c | 794 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 794 insertions(+) create mode 100644 qubes-rpc/tar2qfile.c diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c new file mode 100644 index 0000000..56f7d6e --- /dev/null +++ b/qubes-rpc/tar2qfile.c @@ -0,0 +1,794 @@ +/* +/* $OpenBSD: tar.h,v 1.7 2003/06/02 23:32:09 millert Exp $ */ +/* $NetBSD: tar.h,v 1.3 1995/03/21 09:07:51 cgd Exp $ */ + +/*- + * Copyright (c) 1992 Keith Muller. + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Keith Muller of the University of California, San Diego. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)tar.h 8.2 (Berkeley) 4/18/94 + */ + +#define _GNU_SOURCE /* For O_NOFOLLOW. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +/*************************************************** + * Most routines extracted from the PAX project (tar.c...) * + ***************************************************/ + +/* + * BSD PAX global data structures and constants. + */ + +#define MAXBLK 64512 /* MAX blocksize supported (posix SPEC) */ + /* WARNING: increasing MAXBLK past 32256 */ + /* will violate posix spec. */ +#define MAXBLK_POSIX 32256 /* MAX blocksize supported as per POSIX */ +#define BLKMULT 512 /* blocksize must be even mult of 512 bytes */ + /* Don't even think of changing this */ +#define DEVBLK 8192 /* default read blksize for devices */ +#define FILEBLK 10240 /* default read blksize for files */ +#define PAXPATHLEN 3072 /* maximium path length for pax. MUST be */ + + +/* + * defines and data structures common to all tar formats + */ +#define CHK_LEN 8 /* length of checksum field */ +#define TNMSZ 100 /* size of name field */ +#define NULLCNT 2 /* number of null blocks in trailer */ +#define CHK_OFFSET 148 /* start of chksum field */ +#define BLNKSUM 256L /* sum of checksum field using ' ' */ + +/* + * General Defines + */ +#define HEX 16 +#define OCT 8 +#define _PAX_ 1 +#define _TFILE_BASE "paxXXXXXXXXXX" + +/* + * General Macros + */ +#ifndef MIN +#define MIN(a,b) (((a)<(b))?(a):(b)) +#endif +#ifndef MAX +#define MAX(a,b) (((a)>(b))?(a):(b)) +#endif +#define MAJOR(x) major(x) +#define MINOR(x) minor(x) +#define TODEV(x, y) makedev((x), (y)) + + +/* + * Values used in typeflag field in all tar formats + * (only REGTYPE, LNKTYPE and SYMTYPE are used in old bsd tar headers) + */ +#define REGTYPE '0' /* Regular File */ +#define AREGTYPE '\0' /* Regular File */ +#define LNKTYPE '1' /* Link */ +#define SYMTYPE '2' /* Symlink */ +#define CHRTYPE '3' /* Character Special File */ +#define BLKTYPE '4' /* Block Special File */ +#define DIRTYPE '5' /* Directory */ +#define FIFOTYPE '6' /* FIFO */ +#define CONTTYPE '7' /* high perf file */ + +/* + * GNU tar compatibility; + */ +#define LONGLINKTYPE 'K' /* Long Symlink */ +#define LONGNAMETYPE 'L' /* Long File */ +#define EXTHEADERTYPE 'x' /* Extended header */ + +/* + * Pad with a bit mask, much faster than doing a mod but only works on powers + * of 2. Macro below is for block of 512 bytes. + */ +#define TAR_PAD(x) ((512 - ((x) & 511)) & 511) + +/* + * Data Interchange Format - Extended tar header format - POSIX 1003.1-1990 + */ +#define TPFSZ 155 +#define TMAGIC "ustar" /* ustar and a null */ +#define TMAGLEN 6 +#define TVERSION "00" /* 00 and no null */ +#define TVERSLEN 2 + +typedef struct { + char name[TNMSZ]; /* name of entry */ + char mode[8]; /* mode */ + char uid[8]; /* uid */ + char gid[8]; /* gid */ + char size[12]; /* size */ + char mtime[12]; /* modification time */ + char chksum[CHK_LEN]; /* checksum */ + char typeflag; /* type of file. */ + char linkname[TNMSZ]; /* linked to name */ + char magic[TMAGLEN]; /* magic cookie */ + char version[TVERSLEN]; /* version */ + char uname[32]; /* ascii owner name */ + char gname[32]; /* ascii group name */ + char devmajor[8]; /* major device number */ + char devminor[8]; /* minor device number */ + char prefix[TPFSZ]; /* linked to name */ +} HD_USTAR; + + + +/* + * Routines for manipulating headers, trailers: + * asc_ul() + * tar_trail() + * tar_chksm() + * ustar_id() + */ + +static unsigned long tar_chksm (char *, int); +char *gnu_hack_string; /* GNU ././@LongLink hackery */ + +char untrusted_namebuf[MAX_PATH_LENGTH]; + + +/* + * asc_ul() + * convert hex/octal character string into a u_long. We do not have to + * check for overflow! (the headers in all supported formats are not large + * enough to create an overflow). + * NOTE: strings passed to us are NOT TERMINATED. + * Return: + * unsigned long value + */ + +u_long +asc_ul (char *str, int len, int base) +{ + char *stop; + u_long tval = 0; + + stop = str + len; + + /* + * skip over leading blanks and zeros + */ + while ((str < stop) && ((*str == ' ') || (*str == '0'))) + ++str; + + /* + * for each valid digit, shift running value (tval) over to next digit + * and add next digit + */ + if (base == HEX) + { + while (str < stop) + { + if ((*str >= '0') && (*str <= '9')) + tval = (tval << 4) + (*str++ - '0'); + else if ((*str >= 'A') && (*str <= 'F')) + tval = (tval << 4) + 10 + (*str++ - 'A'); + else if ((*str >= 'a') && (*str <= 'f')) + tval = (tval << 4) + 10 + (*str++ - 'a'); + else + break; + } + } + else + { + while ((str < stop) && (*str >= '0') && (*str <= '7')) + tval = (tval << 3) + (*str++ - '0'); + } + return (tval); +} + + +/* + * tar_trail() + * Called to determine if a header block is a valid trailer. We are passed + * the block, the in_sync flag (which tells us we are in resync mode; + * looking for a valid header), and cnt (which starts at zero) which is + * used to count the number of empty blocks we have seen so far. + * Return: + * 0 if a valid trailer, -1 if not a valid trailer, or 1 if the block + * could never contain a header. + */ + +int +tar_trail (char *buf, + int in_resync, int *cnt) +{ + register int i; + + /* + * look for all zero, trailer is two consecutive blocks of zero + */ + for (i = 0; i < BLKMULT; ++i) + { + if (buf[i] != '\0') + break; + } + + /* + * if not all zero it is not a trailer, but MIGHT be a header. + */ + if (i != BLKMULT) + return (-1); + + /* + * When given a zero block, we must be careful! + * If we are not in resync mode, check for the trailer. Have to watch + * out that we do not mis-identify file data as the trailer, so we do + * NOT try to id a trailer during resync mode. During resync mode we + * might as well throw this block out since a valid header can NEVER be + * a block of all 0 (we must have a valid file name). + */ + if (!in_resync && (++*cnt >= NULLCNT)) + return (0); + return (1); +} + + +/* + * tar_chksm() + * calculate the checksum for a tar block counting the checksum field as + * all blanks (BLNKSUM is that value pre-calculated, the sum of 8 blanks). + * NOTE: we use len to short circuit summing 0's on write since we ALWAYS + * pad headers with 0. + * Return: + * unsigned long checksum + */ + +static unsigned long +tar_chksm (char *blk, int len) +{ + char *stop; + char *pt; + unsigned int chksm = BLNKSUM; /* initial value is checksum field sum */ + + /* + * add the part of the block before the checksum field + */ + pt = blk; + stop = blk + CHK_OFFSET; + while (pt < stop) + chksm += (*pt++ & 0xff); + /* + * move past the checksum field and keep going, spec counts the + * checksum field as the sum of 8 blanks (which is pre-computed as + * BLNKSUM). + * ASSUMED: len is greater than CHK_OFFSET. (len is where our 0 padding + * starts, no point in summing zero's) + */ + pt += CHK_LEN; + stop = blk + len; + while (pt < stop) + chksm += (*pt++ & 0xff); + + return chksm; +} + + +/* + * ustar_id() + * determine if a block given to us is a valid ustar header. We have to + * be on the lookout for those pesky blocks of all zero's + * Return: + * 0 if a ustar header, -1 otherwise + */ + +int +ustar_id (char *blk, size_t size) +{ + HD_USTAR *hd; + + if (size < BLKMULT) + return (-1); + hd = (HD_USTAR *) blk; + + /* + * check for block of zero's first, a simple and fast test then check + * ustar magic cookie. We should use TMAGLEN, but some USTAR archive + * programs are fouled up and create archives missing the \0. Last we + * check the checksum. If ok we have to assume it is a valid header. + */ + if (hd->name[0] == '\0') + return (-1); + if (strncmp (hd->magic, TMAGIC, TMAGLEN - 1) != 0) + return (-1); + if (asc_ul (hd->chksum, sizeof (hd->chksum), OCT) != + tar_chksm (blk, BLKMULT)) + return (-1); + return (0); +} + + + + +/* + * Routines for reading tar files + */ + + + +/* +struct file_header { + unsigned int namelen; + unsigned int mode; + unsigned long long filelen; + unsigned int atime; + unsigned int atime_nsec; + unsigned int mtime; + unsigned int mtime_nsec; +}; +*/ + + + +/* + + +0000cc00 76 6d 2d 74 65 6d 70 6c 61 74 65 73 2f 61 72 63 |vm-templates/arc| +0000cc10 68 6c 69 6e 75 78 2d 78 36 34 2d 73 70 65 63 2d |hlinux-x64-spec-| +0000cc20 32 30 31 33 2e 30 30 30 00 00 00 00 00 00 00 00 |2013.000........| +0000cc30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* +0000cc60 00 00 00 00 30 30 30 30 36 36 34 00 30 30 30 31 |....0000664.0001| +0000cc70 37 35 30 00 30 30 30 31 37 35 31 00 30 37 35 30 |750.0001751.0750| +0000cc80 32 32 30 30 30 34 30 00 31 32 32 32 31 35 32 37 |2200040.12221527| +0000cc90 31 34 34 00 30 31 37 33 30 33 00 20 30 00 00 00 |144.017303. 0...| +0000cca0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* +0000cd00 00 75 73 74 61 72 00 30 30 6f 6d 65 00 00 00 00 |.ustar.00ome....| +0000cd10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +0000cd20 00 00 00 00 00 00 00 00 00 6f 6d 65 00 00 00 00 |.........ome....| +0000cd30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +0000cd40 00 00 00 00 00 00 00 00 00 30 30 30 30 30 30 30 |.........0000000| +0000cd50 00 30 30 30 30 30 30 30 00 00 00 00 00 00 00 00 |.0000000........| +0000cd60 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* + +// Source: http://www.mkssoftware.com/docs/man4/pax.4.asp +struct file_header { // PAX header is similar as file_header and can be completely ignored + unsigned char[100] name; + unsigned char[8] mode; + unsigned char[8] uid; // unused + unsigned char[8] gid; // unused + unsigned char[12] size; // 0 if file is a link + unsigned char[12] mtime; + unsigned char[8] chksum; + unsigned char[1] typeflag; + unsigned char[100] linkname; + unsigned char[6] magic; //ustar + unsigned char[2] version; // 00 + unsigned char[32] uname; // unused + unsigned char[32] gname; // unused + unsigned char[8] devmajor; // unused ? + unsigned char[8] devminor; // unused ? + unsigned char[155] prefix; // only used for files > 100 characters. could be unused ? +}; + +enum { + TYPE_REGULAR, //0 + TYPE_ARCHIVE_LINK, //1 + TYPE_SYMLINK, //2 + TYPE_CHARACTER_DEVICE, //3 + TYPE_BLOCK_DEVICE, //4 + TYPE_DIRECTORY, //5 + TYPE_FIFO, //6 + // Other types: + TYPE_EXTENDED_USAGE, //xxxxx + // A-Z are available for custom usage +}; + +// Extended attribute: +// length keyword=value +// atime, charset, comment, gname, linkpath, mtime, path, size, uname + +*/ + + + +enum { + NEED_NOTHING, + NEED_SKIP, + NEED_READ, + NEED_SYNC_TRAIL +}; + + + +/* + * ustar_rd() + * extract the values out of block already determined to be a ustar header. + * store the values in the ARCHD parameter. + * Return: + * 0 + */ + +int +ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb) +{ + + register HD_USTAR *hd; + register char *dest; + register int cnt = 0; + int ret; + /* DISABLED: unused + dev_t devmajor; + dev_t devminor; + */ + + /* + * we only get proper sized buffers + */ + fprintf(stderr,"Checking if valid header\n"); + if (ustar_id (buf, BLKMULT) < 0) + return (-1); + + fprintf(stderr,"Valid header!\n"); + /* DISABLED: Internal to PAX + arcn->org_name = arcn->name; + arcn->sb.st_nlink = 1; + arcn->pat = NULL; + arcn->nlen = 0; + */ + untrusted_hdr->namelen = 0; + + hd = (HD_USTAR *) buf; + + /* + * see if the filename is split into two parts. if, so joint the parts. + * we copy the prefix first and add a / between the prefix and name. + */ + dest = untrusted_namebuf; + if (*(hd->prefix) != '\0') + { + cnt = strlen(strncpy (dest, hd->prefix, + MIN(sizeof (untrusted_namebuf) - 1,TPFSZ+1))); + dest += cnt; + *dest++ = '/'; + cnt++; + } + if (gnu_hack_string) + { + untrusted_hdr->namelen = cnt + strlen(strncpy (dest, gnu_hack_string, + MIN(TNMSZ+1, sizeof (untrusted_namebuf) - cnt))); + free(gnu_hack_string); + gnu_hack_string = NULL; + } else + untrusted_hdr->namelen = cnt + strlen(strncpy (dest, hd->name, + MIN(TNMSZ+1, sizeof (untrusted_namebuf) - cnt))); + + fprintf(stderr,"Retrieved name len: %d\n",untrusted_hdr->namelen); + fprintf(stderr,"Retrieved name: %s\n",untrusted_namebuf); + + /* + * follow the spec to the letter. we should only have mode bits, strip + * off all other crud we may be passed. + */ + sb->st_mode = (mode_t) (asc_ul (hd->mode, sizeof (hd->mode), OCT) & + 0xfff); + untrusted_hdr->mode = sb->st_mode; + + #if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 64 + sb->st_size = (off_t) asc_uqd (hd->size, sizeof (hd->size), OCT); + #else + sb->st_size = (off_t) asc_ul (hd->size, sizeof (hd->size), OCT); + #endif + + untrusted_hdr->filelen = sb->st_size; + untrusted_hdr->atime = (time_t) asc_ul (hd->mtime, sizeof (hd->mtime), OCT); + untrusted_hdr->mtime = untrusted_hdr->atime; + untrusted_hdr->atime_nsec = untrusted_hdr->mtime_nsec = 0; + + sb->st_mtime = (time_t) asc_ul (hd->mtime, sizeof (hd->mtime), OCT); + sb->st_ctime = sb->st_atime = sb->st_mtime; + + + /* + * If we can find the ascii names for gname and uname in the password + * and group files we will use the uid's and gid they bind. Otherwise + * we use the uid and gid values stored in the header. (This is what + * the posix spec wants). + */ + /* DISABLED: unused + hd->gname[sizeof (hd->gname) - 1] = '\0'; + if (gid_name (hd->gname, &(arcn->sb.st_gid)) < 0) + arcn->sb.st_gid = (gid_t) asc_ul (hd->gid, sizeof (hd->gid), OCT); + hd->uname[sizeof (hd->uname) - 1] = '\0'; + if (uid_name (hd->uname, &(arcn->sb.st_uid)) < 0) + arcn->sb.st_uid = (uid_t) asc_ul (hd->uid, sizeof (hd->uid), OCT); + */ + + /* + * set the defaults, these may be changed depending on the file type + */ + /* Disabled: pax specific + arcn->ln_name[0] = '\0'; + arcn->ln_nlen = 0; + arcn->pad = 0; + arcn->skip = 0; + arcn->sb.st_rdev = (dev_t) 0; + */ + + + /* + * set the mode and PAX type according to the typeflag in the header + */ + switch (hd->typeflag) + { + case FIFOTYPE: + fprintf(stderr,"File is FIFOTYPE\n"); + /* DISABLED: unused + arcn->type = PAX_FIF; + arcn->sb.st_mode |= S_IFIFO; + */ + break; + case DIRTYPE: + fprintf(stderr,"File is DIRTYPE\n"); + /* DISABLED: unused + arcn->type = PAX_DIR; + arcn->sb.st_mode |= S_IFDIR; + arcn->sb.st_nlink = 2; + */ + /* + * Some programs that create ustar archives append a '/' + * to the pathname for directories. This clearly violates + * ustar specs, but we will silently strip it off anyway. + */ +/* + if (arcn->name[arcn->nlen - 1] == '/') + arcn->name[--arcn->nlen] = '\0'; +*/ + break; + case BLKTYPE: + fprintf(stderr,"File is BLKTYPE\n"); + break; + case CHRTYPE: + fprintf(stderr,"File is CHRTYPE\n"); + /* + * this type requires the rdev field to be set. + */ + if (hd->typeflag == BLKTYPE) + { +/* + arcn->type = PAX_BLK; + arcn->sb.st_mode |= S_IFBLK; +*/ + } + else + { +/* + arcn->type = PAX_CHR; + arcn->sb.st_mode |= S_IFCHR; +*/ + } + /* DISABLED: unused + devmajor = (dev_t) asc_ul (hd->devmajor, sizeof (hd->devmajor), OCT); + devminor = (dev_t) asc_ul (hd->devminor, sizeof (hd->devminor), OCT); + */ +// arcn->sb.st_rdev = TODEV (devmajor, devminor); + break; + case SYMTYPE: + fprintf(stderr,"File is SYMTYPE\n"); + break; + case LNKTYPE: + fprintf(stderr,"File is LNKTYPE\n"); + if (hd->typeflag == SYMTYPE) + { +// arcn->type = PAX_SLK; +// arcn->sb.st_mode |= S_IFLNK; + } + else + { +// arcn->type = PAX_HLK; + /* + * so printing looks better + */ +// arcn->sb.st_mode |= S_IFREG; +// arcn->sb.st_nlink = 2; + } + /* + * copy the link name + */ +// arcn->ln_nlen = strlcpy (arcn->ln_name, hd->linkname, +// MIN(TNMSZ+1,sizeof (arcn->ln_name))); + break; + case LONGLINKTYPE: + fprintf(stderr,"File is LONGLINKTYPE\n"); + break; + case LONGNAMETYPE: + fprintf(stderr,"File is LONGNAMETYPE\n"); + /* + * GNU long link/file; we tag these here and let the + * pax internals deal with it -- too ugly otherwise. + */ +// arcn->type = +// hd->typeflag == LONGLINKTYPE ? PAX_GLL : PAX_GLF; +// arcn->pad = TAR_PAD(arcn->sb.st_size); +// arcn->skip = arcn->sb.st_size; +// arcn->ln_name[0] = '\0'; +// arcn->ln_nlen = 0; + break; + case CONTTYPE: + fprintf(stderr,"File is CONTTYPE\n"); + break; + case AREGTYPE: + fprintf(stderr,"File is AREGTYPE\n"); + break; + case REGTYPE: + fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); + write_headers(untrusted_hdr, untrusted_namebuf); + ret = copy_file(1, fd, untrusted_hdr->filelen, &crc32_sum); + if (ret != COPY_FILE_OK) { + if (ret != COPY_FILE_WRITE_ERROR) + gui_fatal("Copying file %s: %s", untrusted_namebuf, + copy_file_status_to_str(ret)); + else { + set_block(0); + wait_for_result(); + exit(1); + } + } + // Extract extra padding + fprintf(stderr,"Need to remove pad:%d %d\n",sb->st_size,BLKMULT-(sb->st_size%BLKMULT)); + ret = read(fd, buf, BLKMULT-(sb->st_size%BLKMULT)); + fprintf(stderr,"Removed %d bytes of padding\n",ret); + + // Resync trailing headers + return NEED_SYNC_TRAIL; + + break; + case EXTHEADERTYPE: + fprintf(stderr,"Extended HEADER encountered\n"); + + fprintf(stderr,"Need to skip %d bytes. Skipping a full block\n",sb->st_size); + sb->st_size = BLKMULT; + while (sb->st_size > 0) { + sb->st_size -= read(fd, buf, MAX(sb->st_size,BLKMULT)); + fprintf(stderr,"now at %d\n",sb->st_size); + } + + + break; + default: + fprintf(stderr,"Default type detected:%c\n",hd->typeflag); + return NEED_SKIP; + /* + * these types have file data that follows. Set the skip and + * pad fields. + */ +// arcn->type = PAX_REG; +// arcn->pad = TAR_PAD (arcn->sb.st_size); +// arcn->skip = arcn->sb.st_size; +// arcn->sb.st_mode |= S_IFREG; + break; + } + return NEED_READ; +} + + + +int tar_file_processor(int fd) +{ + int ret; + int i; + int current; + + struct file_header hdr; + struct stat sb; /* stat buffer see stat(2) */ + + char buf[BLKMULT+1]; + size_t size; + + i=0; + current = NEED_READ; + long sync_count = 0; + while (size = read(fd, &buf, BLKMULT)) { + fprintf(stderr,"Read %d bytes\n",size); + + ret = 0; + if (current==NEED_SYNC_TRAIL) { + ret = tar_trail (buf, 1, &sync_count); + fprintf(stderr,"Synchronizing trail: %d %d\n",ret,sync_count); + if (ret != 1) { + current = NEED_READ; + } + } + if (current==NEED_READ) { + current = ustar_rd(fd, &hdr, &buf, &sb); + fprintf(stderr,"Return %d\n",ret); + } + i++; + //if (i >= 10) + // exit(0); + } + +} + +int main(int argc, char **argv) +{ + int i; + char *entry; + char *cwd; + char *sep; + int fd; + + //signal(SIGPIPE, SIG_IGN); + // this will allow checking for possible feedback packet in the middle of transfer + //set_nonblock(0); + crc32_sum = 0; + cwd = getcwd(NULL, 0); + for (i = 1; i < argc; i++) { + + if (strcmp(argv[i], "--ignore-symlinks")==0) { + ignore_symlinks = 1; + continue; + } else { + // Parse tar file + entry = argv[i]; + fprintf(stderr,"Parsing file %s\n",entry); + + fd = open(entry, O_RDONLY); + if (fd < 0) + fprintf(stderr,"Error opening file %s\n",entry); + + tar_file_processor(fd); + } + } + + if (i <= 1) { + // No argument specified. Use STDIN + fprintf(stderr,"Using STDIN"); + tar_file_processor(fileno(stdin)); + } + + return 0; +} + + + + + + From a05e21ee5f0d1a4e6fe1ca0dbdc1d9b6414a4a19 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 4 Oct 2013 16:26:44 +0200 Subject: [PATCH 05/32] tar2qfile: improved to skip everything but regular files --- qubes-rpc/tar2qfile.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 56f7d6e..cb2fb4b 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -683,14 +683,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s case EXTHEADERTYPE: fprintf(stderr,"Extended HEADER encountered\n"); - fprintf(stderr,"Need to skip %d bytes. Skipping a full block\n",sb->st_size); - sb->st_size = BLKMULT; - while (sb->st_size > 0) { - sb->st_size -= read(fd, buf, MAX(sb->st_size,BLKMULT)); - fprintf(stderr,"now at %d\n",sb->st_size); - } - - + return NEED_SKIP; break; default: fprintf(stderr,"Default type detected:%c\n",hd->typeflag); @@ -705,7 +698,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // arcn->sb.st_mode |= S_IFREG; break; } - return NEED_READ; + return NEED_SKIP; } @@ -724,7 +717,8 @@ int tar_file_processor(int fd) i=0; current = NEED_READ; - long sync_count = 0; + size_t to_skip = 0; + int sync_count = 0; while (size = read(fd, &buf, BLKMULT)) { fprintf(stderr,"Read %d bytes\n",size); @@ -740,6 +734,19 @@ int tar_file_processor(int fd) current = ustar_rd(fd, &hdr, &buf, &sb); fprintf(stderr,"Return %d\n",ret); } + if (current==NEED_SKIP) { + fprintf(stderr,"Need to skip %d bytes\n",sb.st_size); + to_skip = sb.st_size; + while (to_skip > 0) { + to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); + } + + // Extract extra padding + fprintf(stderr,"Need to remove pad:%d %d %d\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); + ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); + fprintf(stderr,"Removed %d bytes of padding\n",ret); + current = NEED_READ; + } i++; //if (i >= 10) // exit(0); From 7efeb57ff288d8de26a0cda1dbc878c2796ed28f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 10 Oct 2013 08:36:24 +0200 Subject: [PATCH 06/32] rpc: moved most of the qfile-agent code to qfile-utils.h/.c --- qubes-rpc/qfile-utils.c | 184 ++++++++++++++++++++++++++++++++++++++++ qubes-rpc/qfile-utils.h | 40 +++++++++ 2 files changed, 224 insertions(+) create mode 100644 qubes-rpc/qfile-utils.c create mode 100644 qubes-rpc/qfile-utils.h diff --git a/qubes-rpc/qfile-utils.c b/qubes-rpc/qfile-utils.c new file mode 100644 index 0000000..6c1a895 --- /dev/null +++ b/qubes-rpc/qfile-utils.c @@ -0,0 +1,184 @@ + +#include + +ignore_symlinks = 0; + +void notify_progress(int size, int flag) +{ + static long long total = 0; + static long long prev_total = 0; + total += size; + if (total > prev_total + PROGRESS_NOTIFY_DELTA + || (flag != PROGRESS_FLAG_NORMAL)) { + // check for possible error from qfile-unpacker; if error occured, + // exit() will be called, so don't bother with current state + // (notify_progress can be called as callback from copy_file()) + if (flag == PROGRESS_FLAG_NORMAL) + wait_for_result(); + do_notify_progress(total, flag); + prev_total = total; + } +} + +void do_notify_progress(long long total, int flag) +{ + char *du_size_env = getenv("FILECOPY_TOTAL_SIZE"); + char *progress_type_env = getenv("PROGRESS_TYPE"); + char *saved_stdout_env = getenv("SAVED_FD_1"); + if (!progress_type_env) + return; + if (!strcmp(progress_type_env, "console") && du_size_env) { + char msg[256]; + snprintf(msg, sizeof(msg), "sent %lld/%lld KB\r", + total / 1024, strtoull(du_size_env, NULL, 0)); + write(2, msg, strlen(msg)); + if (flag == PROGRESS_FLAG_DONE) + write(2, "\n", 1); + } + if (!strcmp(progress_type_env, "gui") && saved_stdout_env) { + char msg[256]; + snprintf(msg, sizeof(msg), "%lld\n", total); + write(strtoul(saved_stdout_env, NULL, 0), msg, + strlen(msg)); + } +} + +void notify_end_and_wait_for_result() +{ + struct file_header end_hdr; + + /* nofity end of transfer */ + memset(&end_hdr, 0, sizeof(end_hdr)); + end_hdr.namelen = 0; + end_hdr.filelen = 0; + write_all_with_crc(1, &end_hdr, sizeof(end_hdr)); + + set_block(0); + wait_for_result(); +} + +int write_all_with_crc(int fd, void *buf, int size) +{ + crc32_sum = Crc32_ComputeBuf(crc32_sum, buf, size); + return write_all(fd, buf, size); +} + +void wait_for_result() +{ + struct result_header hdr; + struct result_header_ext hdr_ext; + char last_filename[MAX_PATH_LENGTH + 1]; + char last_filename_prefix[] = "; Last file: "; + + if (!read_all(0, &hdr, sizeof(hdr))) { + if (errno == EAGAIN) { + // no result sent and stdin still open + return; + } else { + // other read error or EOF + exit(1); // hopefully remote has produced error message + } + } + if (!read_all(0, &hdr_ext, sizeof(hdr_ext))) { + // remote used old result_header struct + hdr_ext.last_namelen = 0; + } + if (hdr_ext.last_namelen > MAX_PATH_LENGTH) { + // read only at most MAX_PATH_LENGTH chars + hdr_ext.last_namelen = MAX_PATH_LENGTH; + } + if (!read_all(0, last_filename, hdr_ext.last_namelen)) { + fprintf(stderr, "Failed to get last filename\n"); + hdr_ext.last_namelen = 0; + } + last_filename[hdr_ext.last_namelen] = '\0'; + if (!hdr_ext.last_namelen) + /* set prefix to empty string */ + last_filename_prefix[0] = '\0'; + + errno = hdr.error_code; + if (hdr.error_code != 0) { + switch (hdr.error_code) { + case EEXIST: + gui_fatal("File copy: not overwriting existing file. Clean QubesIncoming dir, and retry copy%s%s", last_filename_prefix, last_filename); + break; + case EINVAL: + gui_fatal("File copy: Corrupted data from packer%s%s", last_filename_prefix, last_filename); + break; + default: + gui_fatal("File copy: %s%s%s", + strerror(hdr.error_code), last_filename_prefix, last_filename); + } + } + if (hdr.crc32 != crc32_sum) { + gui_fatal("File transfer failed: checksum mismatch"); + } +} + +void write_headers(struct file_header *hdr, char *filename) +{ + if (!write_all_with_crc(1, hdr, sizeof(*hdr)) + || !write_all_with_crc(1, filename, hdr->namelen)) { + set_block(0); + wait_for_result(); + exit(1); + } +} + +int single_file_processor(char *filename, struct stat *st) +{ + struct file_header hdr; + int fd; + mode_t mode = st->st_mode; + + hdr.namelen = strlen(filename) + 1; + hdr.mode = mode; + hdr.atime = st->st_atim.tv_sec; + hdr.atime_nsec = st->st_atim.tv_nsec; + hdr.mtime = st->st_mtim.tv_sec; + hdr.mtime_nsec = st->st_mtim.tv_nsec; + + if (S_ISREG(mode)) { + int ret; + fd = open(filename, O_RDONLY); + if (fd < 0) + gui_fatal("open %s", filename); + hdr.filelen = st->st_size; + write_headers(&hdr, filename); + ret = copy_file(1, fd, hdr.filelen, &crc32_sum); + if (ret != COPY_FILE_OK) { + if (ret != COPY_FILE_WRITE_ERROR) + gui_fatal("Copying file %s: %s", filename, + copy_file_status_to_str(ret)); + else { + set_block(0); + wait_for_result(); + exit(1); + } + } + close(fd); + } + if (S_ISDIR(mode)) { + hdr.filelen = 0; + write_headers(&hdr, filename); + } + if (S_ISLNK(mode) && !ignore_symlinks) { + char name[st->st_size + 1]; + if (readlink(filename, name, sizeof(name)) != st->st_size) + gui_fatal("readlink %s", filename); + hdr.filelen = st->st_size + 1; + write_headers(&hdr, filename); + if (!write_all_with_crc(1, name, st->st_size + 1)) { + set_block(0); + wait_for_result(); + exit(1); + } + } + // check for possible error from qfile-unpacker + wait_for_result(); + return 0; +} + + + + diff --git a/qubes-rpc/qfile-utils.h b/qubes-rpc/qfile-utils.h new file mode 100644 index 0000000..3bb5b18 --- /dev/null +++ b/qubes-rpc/qfile-utils.h @@ -0,0 +1,40 @@ + +#ifndef _LIBQUBES_QFILE_UTILS_H +#define _LIBQUBES_QFILE_UTILS_H 1 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + PROGRESS_FLAG_NORMAL, + PROGRESS_FLAG_INIT, + PROGRESS_FLAG_DONE +}; + +unsigned long crc32_sum; +int ignore_symlinks; + +void notify_progress(int size, int flag); +void do_notify_progress(long long total, int flag); +void notify_end_and_wait_for_result(); + +void write_headers(struct file_header *hdr, char *filename); + +int write_all_with_crc(int fd, void *buf, int size); + +int single_file_processor(char *filename, struct stat *st); + +void wait_for_result(); + +#endif /* _LIBQUBES_QFILE_UTILS_H */ From fb2f6688e8f647033a755582520ff0f48513b2ed Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 10 Oct 2013 08:46:57 +0200 Subject: [PATCH 07/32] rpc: changed qfile agents to use qfile-utils.h --- qubes-rpc/Makefile | 8 +- qubes-rpc/qfile-agent.c | 212 ++-------------------------------------- qubes-rpc/tar2qfile.c | 59 ++--------- 3 files changed, 21 insertions(+), 258 deletions(-) diff --git a/qubes-rpc/Makefile b/qubes-rpc/Makefile index 4f153b2..314e856 100644 --- a/qubes-rpc/Makefile +++ b/qubes-rpc/Makefile @@ -1,14 +1,16 @@ CC=gcc CFLAGS=-g -Wall -I. -fPIC -pie -all: vm-file-editor qopen-in-vm qfile-agent qfile-unpacker +all: vm-file-editor qopen-in-vm qfile-unpacker tar2qfile vm-file-editor: vm-file-editor.o ioall.o $(CC) -pie -g -o $@ $^ qopen-in-vm: qopen-in-vm.o ioall.o gui-fatal.o $(CC) -pie -g -o $@ $^ -qfile-agent: qfile-agent.o gui-fatal.o +qfile-agent: qfile-agent.o gui-fatal.o qfile-utils.o $(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy qfile-unpacker: qfile-unpacker.o gui-fatal.o $(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy +tar2qfile: qfile-utils.o tar2qfile.o gui-fatal.o + $(CC) -pie -g -o $@ $^ -lqubes-rpc-filecopy clean: - rm -f qopen-in-vm qfile-agent qfile-unpacker vm-file-editor *.o *~ + rm -f qopen-in-vm qfile-agent qfile-unpacker tar2qfile vm-file-editor *.o *~ diff --git a/qubes-rpc/qfile-agent.c b/qubes-rpc/qfile-agent.c index 22712c7..f4c3586 100644 --- a/qubes-rpc/qfile-agent.c +++ b/qubes-rpc/qfile-agent.c @@ -1,186 +1,15 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -enum { - PROGRESS_FLAG_NORMAL, - PROGRESS_FLAG_INIT, - PROGRESS_FLAG_DONE -}; +#include "qfile-utils.h" -int ignore_symlinks = 0; +ignore_symlinks = 0; -unsigned long crc32_sum; -int write_all_with_crc(int fd, void *buf, int size) +char *get_abs_path(char *cwd, char *pathname) { - crc32_sum = Crc32_ComputeBuf(crc32_sum, buf, size); - return write_all(fd, buf, size); -} - -void do_notify_progress(long long total, int flag) -{ - char *du_size_env = getenv("FILECOPY_TOTAL_SIZE"); - char *progress_type_env = getenv("PROGRESS_TYPE"); - char *saved_stdout_env = getenv("SAVED_FD_1"); - if (!progress_type_env) - return; - if (!strcmp(progress_type_env, "console") && du_size_env) { - char msg[256]; - snprintf(msg, sizeof(msg), "sent %lld/%lld KB\r", - total / 1024, strtoull(du_size_env, NULL, 0)); - write(2, msg, strlen(msg)); - if (flag == PROGRESS_FLAG_DONE) - write(2, "\n", 1); - } - if (!strcmp(progress_type_env, "gui") && saved_stdout_env) { - char msg[256]; - snprintf(msg, sizeof(msg), "%lld\n", total); - write(strtoul(saved_stdout_env, NULL, 0), msg, - strlen(msg)); - } -} - -void wait_for_result() -{ - struct result_header hdr; - struct result_header_ext hdr_ext; - char last_filename[MAX_PATH_LENGTH + 1]; - char last_filename_prefix[] = "; Last file: "; - - if (!read_all(0, &hdr, sizeof(hdr))) { - if (errno == EAGAIN) { - // no result sent and stdin still open - return; - } else { - // other read error or EOF - exit(1); // hopefully remote has produced error message - } - } - if (!read_all(0, &hdr_ext, sizeof(hdr_ext))) { - // remote used old result_header struct - hdr_ext.last_namelen = 0; - } - if (hdr_ext.last_namelen > MAX_PATH_LENGTH) { - // read only at most MAX_PATH_LENGTH chars - hdr_ext.last_namelen = MAX_PATH_LENGTH; - } - if (!read_all(0, last_filename, hdr_ext.last_namelen)) { - fprintf(stderr, "Failed to get last filename\n"); - hdr_ext.last_namelen = 0; - } - last_filename[hdr_ext.last_namelen] = '\0'; - if (!hdr_ext.last_namelen) - /* set prefix to empty string */ - last_filename_prefix[0] = '\0'; - - errno = hdr.error_code; - if (hdr.error_code != 0) { - switch (hdr.error_code) { - case EEXIST: - gui_fatal("File copy: not overwriting existing file. Clean QubesIncoming dir, and retry copy%s%s", last_filename_prefix, last_filename); - break; - case EINVAL: - gui_fatal("File copy: Corrupted data from packer%s%s", last_filename_prefix, last_filename); - break; - default: - gui_fatal("File copy: %s%s%s", - strerror(hdr.error_code), last_filename_prefix, last_filename); - } - } - if (hdr.crc32 != crc32_sum) { - gui_fatal("File transfer failed: checksum mismatch"); - } -} - -void notify_progress(int size, int flag) -{ - static long long total = 0; - static long long prev_total = 0; - total += size; - if (total > prev_total + PROGRESS_NOTIFY_DELTA - || (flag != PROGRESS_FLAG_NORMAL)) { - // check for possible error from qfile-unpacker; if error occured, - // exit() will be called, so don't bother with current state - // (notify_progress can be called as callback from copy_file()) - if (flag == PROGRESS_FLAG_NORMAL) - wait_for_result(); - do_notify_progress(total, flag); - prev_total = total; - } -} - -void write_headers(struct file_header *hdr, char *filename) -{ - if (!write_all_with_crc(1, hdr, sizeof(*hdr)) - || !write_all_with_crc(1, filename, hdr->namelen)) { - set_block(0); - wait_for_result(); - exit(1); - } -} - -int single_file_processor(char *filename, struct stat *st) -{ - struct file_header hdr; - int fd; - mode_t mode = st->st_mode; - - hdr.namelen = strlen(filename) + 1; - hdr.mode = mode; - hdr.atime = st->st_atim.tv_sec; - hdr.atime_nsec = st->st_atim.tv_nsec; - hdr.mtime = st->st_mtim.tv_sec; - hdr.mtime_nsec = st->st_mtim.tv_nsec; - - if (S_ISREG(mode)) { - int ret; - fd = open(filename, O_RDONLY); - if (fd < 0) - gui_fatal("open %s", filename); - hdr.filelen = st->st_size; - write_headers(&hdr, filename); - ret = copy_file(1, fd, hdr.filelen, &crc32_sum); - if (ret != COPY_FILE_OK) { - if (ret != COPY_FILE_WRITE_ERROR) - gui_fatal("Copying file %s: %s", filename, - copy_file_status_to_str(ret)); - else { - set_block(0); - wait_for_result(); - exit(1); - } - } - close(fd); - } - if (S_ISDIR(mode)) { - hdr.filelen = 0; - write_headers(&hdr, filename); - } - if (S_ISLNK(mode) && !ignore_symlinks) { - char name[st->st_size + 1]; - if (readlink(filename, name, sizeof(name)) != st->st_size) - gui_fatal("readlink %s", filename); - hdr.filelen = st->st_size + 1; - write_headers(&hdr, filename); - if (!write_all_with_crc(1, name, st->st_size + 1)) { - set_block(0); - wait_for_result(); - exit(1); - } - } - // check for possible error from qfile-unpacker - wait_for_result(); - return 0; + char *ret; + if (pathname[0] == '/') + return strdup(pathname); + asprintf(&ret, "%s/%s", cwd, pathname); + return ret; } int do_fs_walk(char *file) @@ -213,29 +42,6 @@ int do_fs_walk(char *file) return 0; } -void notify_end_and_wait_for_result() -{ - struct file_header end_hdr; - - /* nofity end of transfer */ - memset(&end_hdr, 0, sizeof(end_hdr)); - end_hdr.namelen = 0; - end_hdr.filelen = 0; - write_all_with_crc(1, &end_hdr, sizeof(end_hdr)); - - set_block(0); - wait_for_result(); -} - -char *get_abs_path(char *cwd, char *pathname) -{ - char *ret; - if (pathname[0] == '/') - return strdup(pathname); - asprintf(&ret, "%s/%s", cwd, pathname); - return ret; -} - int main(int argc, char **argv) { int i; @@ -275,3 +81,5 @@ int main(int argc, char **argv) notify_progress(0, PROGRESS_FLAG_DONE); return 0; } + + diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index cb2fb4b..1965223 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -47,8 +47,7 @@ #include #include #include -#include - +#include /*************************************************** @@ -157,7 +156,6 @@ typedef struct { } HD_USTAR; - /* * Routines for manipulating headers, trailers: * asc_ul() @@ -343,50 +341,8 @@ ustar_id (char *blk, size_t size) } - - /* * Routines for reading tar files - */ - - - -/* -struct file_header { - unsigned int namelen; - unsigned int mode; - unsigned long long filelen; - unsigned int atime; - unsigned int atime_nsec; - unsigned int mtime; - unsigned int mtime_nsec; -}; -*/ - - - -/* - - -0000cc00 76 6d 2d 74 65 6d 70 6c 61 74 65 73 2f 61 72 63 |vm-templates/arc| -0000cc10 68 6c 69 6e 75 78 2d 78 36 34 2d 73 70 65 63 2d |hlinux-x64-spec-| -0000cc20 32 30 31 33 2e 30 30 30 00 00 00 00 00 00 00 00 |2013.000........| -0000cc30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -* -0000cc60 00 00 00 00 30 30 30 30 36 36 34 00 30 30 30 31 |....0000664.0001| -0000cc70 37 35 30 00 30 30 30 31 37 35 31 00 30 37 35 30 |750.0001751.0750| -0000cc80 32 32 30 30 30 34 30 00 31 32 32 32 31 35 32 37 |2200040.12221527| -0000cc90 31 34 34 00 30 31 37 33 30 33 00 20 30 00 00 00 |144.017303. 0...| -0000cca0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -* -0000cd00 00 75 73 74 61 72 00 30 30 6f 6d 65 00 00 00 00 |.ustar.00ome....| -0000cd10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -0000cd20 00 00 00 00 00 00 00 00 00 6f 6d 65 00 00 00 00 |.........ome....| -0000cd30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -0000cd40 00 00 00 00 00 00 00 00 00 30 30 30 30 30 30 30 |.........0000000| -0000cd50 00 30 30 30 30 30 30 30 00 00 00 00 00 00 00 00 |.0000000........| -0000cd60 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -* // Source: http://www.mkssoftware.com/docs/man4/pax.4.asp struct file_header { // PAX header is similar as file_header and can be completely ignored @@ -428,7 +384,6 @@ enum { */ - enum { NEED_NOTHING, NEED_SKIP, @@ -437,7 +392,6 @@ enum { }; - /* * ustar_rd() * extract the values out of block already determined to be a ustar header. @@ -721,7 +675,6 @@ int tar_file_processor(int fd) int sync_count = 0; while (size = read(fd, &buf, BLKMULT)) { fprintf(stderr,"Read %d bytes\n",size); - ret = 0; if (current==NEED_SYNC_TRAIL) { ret = tar_trail (buf, 1, &sync_count); @@ -765,6 +718,8 @@ int main(int argc, char **argv) //signal(SIGPIPE, SIG_IGN); // this will allow checking for possible feedback packet in the middle of transfer //set_nonblock(0); + notify_progress(0, PROGRESS_FLAG_INIT); + crc32_sum = 0; cwd = getcwd(NULL, 0); for (i = 1; i < argc; i++) { @@ -787,15 +742,13 @@ int main(int argc, char **argv) if (i <= 1) { // No argument specified. Use STDIN - fprintf(stderr,"Using STDIN"); + fprintf(stderr,"Using STDIN\n"); tar_file_processor(fileno(stdin)); } + notify_end_and_wait_for_result(); + notify_progress(0, PROGRESS_FLAG_DONE); return 0; } - - - - From 6938e68ee6a398ebef424e8c4107a4f1e71ef397 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 10 Oct 2013 08:52:10 +0200 Subject: [PATCH 08/32] tar2qfile: improved error handling --- qubes-rpc/tar2qfile.c | 63 +++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 1965223..0ce899d 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -388,7 +388,8 @@ enum { NEED_NOTHING, NEED_SKIP, NEED_READ, - NEED_SYNC_TRAIL + NEED_SYNC_TRAIL, + INVALID_HEADER, }; @@ -417,9 +418,10 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s * we only get proper sized buffers */ fprintf(stderr,"Checking if valid header\n"); - if (ustar_id (buf, BLKMULT) < 0) - return (-1); - + if (ustar_id (buf, BLKMULT) < 0) { + fprintf (stderr,"Invalid header\n"); + return INVALID_HEADER; + } fprintf(stderr,"Valid header!\n"); /* DISABLED: Internal to PAX arcn->org_name = arcn->name; @@ -674,33 +676,35 @@ int tar_file_processor(int fd) size_t to_skip = 0; int sync_count = 0; while (size = read(fd, &buf, BLKMULT)) { - fprintf(stderr,"Read %d bytes\n",size); - ret = 0; - if (current==NEED_SYNC_TRAIL) { - ret = tar_trail (buf, 1, &sync_count); - fprintf(stderr,"Synchronizing trail: %d %d\n",ret,sync_count); - if (ret != 1) { - current = NEED_READ; + if (size != -1) { + fprintf(stderr,"Read %d bytes\n",size); + ret = 0; + if (current==NEED_SYNC_TRAIL) { + ret = tar_trail (buf, 1, &sync_count); + fprintf(stderr,"Synchronizing trail: %d %d\n",ret,sync_count); + if (ret != 1) { + current = NEED_READ; + } } - } - if (current==NEED_READ) { - current = ustar_rd(fd, &hdr, &buf, &sb); - fprintf(stderr,"Return %d\n",ret); - } - if (current==NEED_SKIP) { - fprintf(stderr,"Need to skip %d bytes\n",sb.st_size); - to_skip = sb.st_size; - while (to_skip > 0) { - to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); + if (current==NEED_READ) { + current = ustar_rd(fd, &hdr, &buf, &sb); + fprintf(stderr,"Return %d\n",ret); } + if (current==NEED_SKIP) { + fprintf(stderr,"Need to skip %d bytes\n",sb.st_size); + to_skip = sb.st_size; + while (to_skip > 0) { + to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); + } - // Extract extra padding - fprintf(stderr,"Need to remove pad:%d %d %d\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); - ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); - fprintf(stderr,"Removed %d bytes of padding\n",ret); - current = NEED_READ; - } - i++; + // Extract extra padding + fprintf(stderr,"Need to remove pad:%d %d %d\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); + ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); + fprintf(stderr,"Removed %d bytes of padding\n",ret); + current = NEED_READ; + } + i++; + } //if (i >= 10) // exit(0); } @@ -717,7 +721,7 @@ int main(int argc, char **argv) //signal(SIGPIPE, SIG_IGN); // this will allow checking for possible feedback packet in the middle of transfer - //set_nonblock(0); + // set_nonblock(0); notify_progress(0, PROGRESS_FLAG_INIT); crc32_sum = 0; @@ -743,6 +747,7 @@ int main(int argc, char **argv) if (i <= 1) { // No argument specified. Use STDIN fprintf(stderr,"Using STDIN\n"); + set_block(0); tar_file_processor(fileno(stdin)); } From 864118cf10a64dbc45561d29b9952211d339b88a Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 10 Oct 2013 08:54:44 +0200 Subject: [PATCH 09/32] tar2qfile: multiples fixes to match the qfile format requirements --- qubes-rpc/tar2qfile.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 0ce899d..7c3ddfa 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -456,6 +456,9 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s untrusted_hdr->namelen = cnt + strlen(strncpy (dest, hd->name, MIN(TNMSZ+1, sizeof (untrusted_namebuf) - cnt))); + // qfile count the \0 in the namelen + untrusted_hdr->namelen += 1; + fprintf(stderr,"Retrieved name len: %d\n",untrusted_hdr->namelen); fprintf(stderr,"Retrieved name: %s\n",untrusted_namebuf); @@ -615,6 +618,8 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s break; case REGTYPE: fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); + // Restored POSIX stat file mode (because PAX format use its own file type) + untrusted_hdr->mode |= S_IFREG; write_headers(untrusted_hdr, untrusted_namebuf); ret = copy_file(1, fd, untrusted_hdr->filelen, &crc32_sum); if (ret != COPY_FILE_OK) { From d1559c54132481e01fada4334a8f2a433a978993 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 16:02:16 +0200 Subject: [PATCH 10/32] tar2qfile: send directories headers even if the tar file does not contain any directory headers --- qubes-rpc/tar2qfile.c | 108 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 7 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 7c3ddfa..7fab031 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -49,7 +49,6 @@ #include #include - /*************************************************** * Most routines extracted from the PAX project (tar.c...) * ***************************************************/ @@ -390,6 +389,7 @@ enum { NEED_READ, NEED_SYNC_TRAIL, INVALID_HEADER, + MEMORY_ALLOC_FAILED, }; @@ -401,6 +401,9 @@ enum { * 0 */ +int n_dirs = 0; +char ** dirs_headers_sent = NULL; + int ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb) { @@ -618,15 +621,104 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s break; case REGTYPE: fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); - // Restored POSIX stat file mode (because PAX format use its own file type) + + // Create a copy of untrusted_namebuf to be used for strtok + char * dirbuf; + dirbuf = malloc(sizeof (char) * (untrusted_hdr->namelen)); + if (dirbuf == NULL) + return MEMORY_ALLOC_FAILED; + dirbuf = strncpy(dirbuf, untrusted_namebuf, untrusted_hdr->namelen); + + int i = 0; + int dir_found = 0; + size_t pathsize = 0; + char * path = NULL; + struct file_header dir_header; + + // Split the path in directories and recompose it incrementally + char * last_token = strtok(dirbuf,"/"); + char * token = strtok(NULL, "/"); + while (token != NULL) { + + fprintf(stderr,"Found directory %s (last:%s)\n",token,last_token); + + // Recompose the path based on last discovered directory + if (path == NULL) { + path = malloc(sizeof (char) * (strlen(last_token)+1)); + if (path == NULL) + return MEMORY_ALLOC_FAILED; + path = strncpy(path, last_token, strlen(last_token)); + path[strlen(last_token)] = '\0'; + } else { + pathsize = strlen(path); + path = realloc(path, sizeof (char) * (strlen(path)+1+strlen(last_token)+1)); + if (path == NULL) + return MEMORY_ALLOC_FAILED; + path[pathsize] = '/'; + + strncpy(path+pathsize+1, last_token, strlen(last_token)); + path[pathsize+strlen(last_token)+1] = '\0'; + } + fprintf(stderr,"Path is %s\n",path); + + fprintf(stderr,"Checking from i=0 i<%d\n",n_dirs); + // Verify if qfile headers for the current path have already been sent based on the dirs_headers_sent table + dir_found = 0; + for (i = 0; i < n_dirs; ++i) { + fprintf(stderr,"Comparing with %d %d %s %s\n",i,n_dirs,dirs_headers_sent[i],path); + if (strcmp(dirs_headers_sent[i],path)==0) { + fprintf(stderr,"Directory headers already sent\n"); + dir_found=1; + } + } + if (dir_found == 0) { + // Register the current path as being sent in the dirs_headers_sent table + fprintf(stderr,"Inserting %s into register\n",path); + dirs_headers_sent = realloc(dirs_headers_sent, sizeof (char*) * n_dirs++); + if (dirs_headers_sent == NULL) + return MEMORY_ALLOC_FAILED; + dirs_headers_sent[n_dirs-1] = malloc(sizeof (char) * (strlen(path)+1)); + if (dirs_headers_sent[n_dirs-1] == NULL) + return MEMORY_ALLOC_FAILED; + strncpy(dirs_headers_sent[n_dirs-1], path, strlen(path)+1); + + // Initialize the qfile headers for the current directory path + dir_header.namelen = strlen(path)+1; + dir_header.atime = untrusted_hdr->atime; + dir_header.atime_nsec = untrusted_hdr->atime_nsec; + dir_header.mtime = untrusted_hdr->mtime; + dir_header.mtime_nsec = untrusted_hdr->mtime_nsec; + + dir_header.mode = untrusted_hdr->mode | S_IFDIR; + dir_header.filelen = 0; + + fprintf(stderr,"Sending directory headers for %s\n",path); + // Send the qfile headers for the current directory path + write_headers(&dir_header, path); + } + last_token = token; + token = strtok(NULL, "/"); + } + free(path); + free(dirbuf); + + fprintf(stderr,"End of directory checks\n"); + + // Restore POSIX stat file mode (because PAX format use its own file type) untrusted_hdr->mode |= S_IFREG; + fprintf(stderr,"Writing file header\n"); + // Send header and file content write_headers(untrusted_hdr, untrusted_namebuf); + fprintf(stderr,"Writing file content\n"); ret = copy_file(1, fd, untrusted_hdr->filelen, &crc32_sum); + + fprintf(stderr,"Copyfile returned with error %d\n",ret); if (ret != COPY_FILE_OK) { if (ret != COPY_FILE_WRITE_ERROR) gui_fatal("Copying file %s: %s", untrusted_namebuf, copy_file_status_to_str(ret)); else { + fprintf(stderr,"UNKNOWN ERROR RETURN STATUS:%d\n.. Waiting...\n",ret); set_block(0); wait_for_result(); exit(1); @@ -637,7 +729,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s ret = read(fd, buf, BLKMULT-(sb->st_size%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); - // Resync trailing headers + // Resync trailing headers in order to find next file chunck in the tar file return NEED_SYNC_TRAIL; break; @@ -724,15 +816,16 @@ int main(int argc, char **argv) char *sep; int fd; - //signal(SIGPIPE, SIG_IGN); + signal(SIGPIPE, SIG_IGN); // this will allow checking for possible feedback packet in the middle of transfer - // set_nonblock(0); + // if disabled, the copy_file process could hang notify_progress(0, PROGRESS_FLAG_INIT); + //set_size_limit(1500000000, 2048); crc32_sum = 0; cwd = getcwd(NULL, 0); for (i = 1; i < argc; i++) { - + set_nonblock(0); if (strcmp(argv[i], "--ignore-symlinks")==0) { ignore_symlinks = 1; continue; @@ -756,7 +849,8 @@ int main(int argc, char **argv) tar_file_processor(fileno(stdin)); } - notify_end_and_wait_for_result(); + + //notify_end_and_wait_for_result(); notify_progress(0, PROGRESS_FLAG_DONE); return 0; } From 429211ade451b1aff6c7f3028c8f1f65533ae960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 18:58:46 +0100 Subject: [PATCH 11/32] Restore qfile-agent compilation Must be removed by mistake... --- qubes-rpc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes-rpc/Makefile b/qubes-rpc/Makefile index 314e856..0a3d5e8 100644 --- a/qubes-rpc/Makefile +++ b/qubes-rpc/Makefile @@ -1,6 +1,6 @@ CC=gcc CFLAGS=-g -Wall -I. -fPIC -pie -all: vm-file-editor qopen-in-vm qfile-unpacker tar2qfile +all: vm-file-editor qopen-in-vm qfile-agent qfile-unpacker tar2qfile vm-file-editor: vm-file-editor.o ioall.o $(CC) -pie -g -o $@ $^ qopen-in-vm: qopen-in-vm.o ioall.o gui-fatal.o From 6eaa30a96a45e0c484925fa90d6dd3d2ed21075b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 19:00:37 +0100 Subject: [PATCH 12/32] qfile-utils: fix global variables declarations --- qubes-rpc/qfile-agent.c | 3 --- qubes-rpc/qfile-utils.c | 3 ++- qubes-rpc/qfile-utils.h | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/qubes-rpc/qfile-agent.c b/qubes-rpc/qfile-agent.c index f4c3586..2eb8205 100644 --- a/qubes-rpc/qfile-agent.c +++ b/qubes-rpc/qfile-agent.c @@ -1,8 +1,5 @@ - #include "qfile-utils.h" -ignore_symlinks = 0; - char *get_abs_path(char *cwd, char *pathname) { char *ret; diff --git a/qubes-rpc/qfile-utils.c b/qubes-rpc/qfile-utils.c index 6c1a895..6e62882 100644 --- a/qubes-rpc/qfile-utils.c +++ b/qubes-rpc/qfile-utils.c @@ -1,7 +1,8 @@ #include -ignore_symlinks = 0; +unsigned long crc32_sum; +int ignore_symlinks = 0; void notify_progress(int size, int flag) { diff --git a/qubes-rpc/qfile-utils.h b/qubes-rpc/qfile-utils.h index 3bb5b18..389c74a 100644 --- a/qubes-rpc/qfile-utils.h +++ b/qubes-rpc/qfile-utils.h @@ -22,8 +22,8 @@ enum { PROGRESS_FLAG_DONE }; -unsigned long crc32_sum; -int ignore_symlinks; +extern unsigned long crc32_sum; +extern int ignore_symlinks; void notify_progress(int size, int flag); void do_notify_progress(long long total, int flag); From 639cb51414b11a8bbb71592e9c79b3a9676dcc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 19:01:57 +0100 Subject: [PATCH 13/32] Add qubes.{Backup,Restore} services, include them in rpm package --- Makefile | 2 ++ qubes-rpc/qubes.Backup | 23 +++++++++++++++++++++++ qubes-rpc/qubes.Restore | 32 ++++++++++++++++++++++++++++++++ rpm_spec/core-vm.spec | 3 +++ 4 files changed, 60 insertions(+) create mode 100644 qubes-rpc/qubes.Backup create mode 100644 qubes-rpc/qubes.Restore diff --git a/Makefile b/Makefile index 36c561e..ac0d382 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,7 @@ install-vm: install qubes-rpc/qvm-copy-to-vm.kde $(DESTDIR)/usr/lib/qubes install qubes-rpc/qvm-copy-to-vm.gnome $(DESTDIR)/usr/lib/qubes install qubes-rpc/{vm-file-editor,qfile-agent,qopen-in-vm} $(DESTDIR)/usr/lib/qubes + install qubes-rpc/tar2qfile $(DESTDIR)/usr/lib/qubes # Install qfile-unpacker as SUID - because it will fail to receive files from other vm install -m 4555 qubes-rpc/qfile-unpacker $(DESTDIR)/usr/lib/qubes install qubes-rpc/qrun-in-vm $(DESTDIR)/usr/lib/qubes @@ -132,6 +133,7 @@ install-vm: install -m 0644 qubes-rpc/{qubes.SuspendPre,qubes.SuspendPost,qubes.GetAppmenus} $(DESTDIR)/etc/qubes-rpc install -m 0644 qubes-rpc/qubes.WaitForSession $(DESTDIR)/etc/qubes-rpc install -m 0644 qubes-rpc/qubes.DetachPciDevice $(DESTDIR)/etc/qubes-rpc + install -m 0644 qubes-rpc/qubes.{Backup,Restore} $(DESTDIR)/etc/qubes-rpc install -d $(DESTDIR)/usr/share/file-manager/actions install -m 0644 qubes-rpc/*-gnome.desktop $(DESTDIR)/usr/share/file-manager/actions diff --git a/qubes-rpc/qubes.Backup b/qubes-rpc/qubes.Backup new file mode 100644 index 0000000..6e3f1d4 --- /dev/null +++ b/qubes-rpc/qubes.Backup @@ -0,0 +1,23 @@ +echo Starting Backupcopy +read args +echo Arguments: $args +if [ -d "$args" ] ; then + echo "Performing backup to directory $args" + TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`" + echo "Copying STDIN data to $TARGET" + cat > $TARGET +else + echo "Checking if arguments is matching a command" + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting STDIN to $args" + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" + exit 1 + fi +fi diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore new file mode 100644 index 0000000..8480ea4 --- /dev/null +++ b/qubes-rpc/qubes.Restore @@ -0,0 +1,32 @@ +echo Starting Restorecopy >2 +read args +echo Arguments: $args >2 +if [ -f "$args" ] ; then + echo "Performing restore from backup file $args" >2 + TARGET="$args" + echo "Copying $TARGET to STDOUT" >2 + /usr/lib/qubes/tar2qfile $TARGET +else + echo "Checking if arguments is matching a command" >2 + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + tmpdir=`mktemp -d` + mkfifo $tmpdir/backup-data + echo "Redirecting $args to STDOUT" >2 + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + # Use named pipe to pass original stdin to tar2file + $@ > $tmpdir/backup-data < /dev/null & + retcode=$? + /usr/lib/qubes/tar2qfile $tmpdir/backup-data + wait + rm $tmpdir/backup-data + rmdir $tmpdir + exit $retcode + else + echo "Invalid command $COMMAND" >2 + exit 1 + fi +fi diff --git a/rpm_spec/core-vm.spec b/rpm_spec/core-vm.spec index 9d5f2fa..1834f66 100644 --- a/rpm_spec/core-vm.spec +++ b/rpm_spec/core-vm.spec @@ -277,6 +277,8 @@ rm -f %{name}-%{version} /etc/qubes-rpc/qubes.SuspendPost /etc/qubes-rpc/qubes.WaitForSession /etc/qubes-rpc/qubes.DetachPciDevice +/etc/qubes-rpc/qubes.Backup +/etc/qubes-rpc/qubes.Restore /etc/sudoers.d/qubes %config(noreplace) /etc/sysconfig/iptables %config(noreplace) /etc/sysconfig/ip6tables @@ -322,6 +324,7 @@ rm -f %{name}-%{version} /usr/lib/qubes/qvm-copy-to-vm.kde /usr/lib/qubes/serial.conf /usr/lib/qubes/setup-ip +/usr/lib/qubes/tar2qfile /usr/lib/qubes/vm-file-editor /usr/lib/qubes/wrap-in-html-if-url.sh /usr/lib/qubes/iptables-yum-proxy From af034251876a7681c5f840d704fead302574252b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 19:02:53 +0100 Subject: [PATCH 14/32] tar2qfile: fix compiler warnings Actually one was real bug: - current = ustar_rd(fd, &hdr, &buf, &sb); + current = ustar_rd(fd, &hdr, buf, &sb); The others was mostly invalid printf format string. --- qubes-rpc/tar2qfile.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 7fab031..d41f2e8 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -1,4 +1,3 @@ -/* /* $OpenBSD: tar.h,v 1.7 2003/06/02 23:32:09 millert Exp $ */ /* $NetBSD: tar.h,v 1.3 1995/03/21 09:07:51 cgd Exp $ */ @@ -620,7 +619,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s fprintf(stderr,"File is AREGTYPE\n"); break; case REGTYPE: - fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); + fprintf(stderr,"File is REGTYPE of size %ld\n",sb->st_size); // Create a copy of untrusted_namebuf to be used for strtok char * dirbuf; @@ -725,7 +724,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } } // Extract extra padding - fprintf(stderr,"Need to remove pad:%d %d\n",sb->st_size,BLKMULT-(sb->st_size%BLKMULT)); + fprintf(stderr,"Need to remove pad:%ld %ld\n",sb->st_size,BLKMULT-(sb->st_size%BLKMULT)); ret = read(fd, buf, BLKMULT-(sb->st_size%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); @@ -756,7 +755,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s -int tar_file_processor(int fd) +void tar_file_processor(int fd) { int ret; int i; @@ -772,9 +771,9 @@ int tar_file_processor(int fd) current = NEED_READ; size_t to_skip = 0; int sync_count = 0; - while (size = read(fd, &buf, BLKMULT)) { + while ((size = read(fd, &buf, BLKMULT))) { if (size != -1) { - fprintf(stderr,"Read %d bytes\n",size); + fprintf(stderr,"Read %ld bytes\n",size); ret = 0; if (current==NEED_SYNC_TRAIL) { ret = tar_trail (buf, 1, &sync_count); @@ -784,18 +783,18 @@ int tar_file_processor(int fd) } } if (current==NEED_READ) { - current = ustar_rd(fd, &hdr, &buf, &sb); + current = ustar_rd(fd, &hdr, buf, &sb); fprintf(stderr,"Return %d\n",ret); } if (current==NEED_SKIP) { - fprintf(stderr,"Need to skip %d bytes\n",sb.st_size); + fprintf(stderr,"Need to skip %ld bytes\n",sb.st_size); to_skip = sb.st_size; while (to_skip > 0) { to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); } // Extract extra padding - fprintf(stderr,"Need to remove pad:%d %d %d\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); + fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); current = NEED_READ; @@ -812,8 +811,6 @@ int main(int argc, char **argv) { int i; char *entry; - char *cwd; - char *sep; int fd; signal(SIGPIPE, SIG_IGN); @@ -823,7 +820,6 @@ int main(int argc, char **argv) //set_size_limit(1500000000, 2048); crc32_sum = 0; - cwd = getcwd(NULL, 0); for (i = 1; i < argc; i++) { set_nonblock(0); if (strcmp(argv[i], "--ignore-symlinks")==0) { From 1bd16d981c8b82aa2b6e1c45b3300647dc623eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 19:04:24 +0100 Subject: [PATCH 15/32] tar2qfile: ignore EDQUOT error from dom0 dom0 will use quota enforcement to extract only backup header, so this is normal situation in this tool. --- qubes-rpc/qfile-utils.c | 8 ++++++++ qubes-rpc/tar2qfile.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/qubes-rpc/qfile-utils.c b/qubes-rpc/qfile-utils.c index 6e62882..ac804f8 100644 --- a/qubes-rpc/qfile-utils.c +++ b/qubes-rpc/qfile-utils.c @@ -3,6 +3,7 @@ unsigned long crc32_sum; int ignore_symlinks = 0; +int ignore_quota_error = 0; void notify_progress(int size, int flag) { @@ -106,6 +107,13 @@ void wait_for_result() case EINVAL: gui_fatal("File copy: Corrupted data from packer%s%s", last_filename_prefix, last_filename); break; + case EDQUOT: + if (ignore_quota_error) { + /* skip also CRC check as sender and receiver might be + * desynchronized in this case */ + return; + } + /* fall though */ default: gui_fatal("File copy: %s%s%s", strerror(hdr.error_code), last_filename_prefix, last_filename); diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index d41f2e8..408150a 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -166,6 +166,7 @@ static unsigned long tar_chksm (char *, int); char *gnu_hack_string; /* GNU ././@LongLink hackery */ char untrusted_namebuf[MAX_PATH_LENGTH]; +extern int ignore_quota_error; /* @@ -820,6 +821,9 @@ int main(int argc, char **argv) //set_size_limit(1500000000, 2048); crc32_sum = 0; + /* when extracting backup header, dom0 will terminate the transfer with + * EDQUOT just after getting qubes.xml */ + ignore_quota_error = 1; for (i = 1; i < argc; i++) { set_nonblock(0); if (strcmp(argv[i], "--ignore-symlinks")==0) { From 08a78d4c321dfebef6b87697edb1a1cfe1d944fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 14 Nov 2013 21:37:16 +0100 Subject: [PATCH 16/32] qvm-open-in-vm: fix path for URL wrapper --- qubes-rpc/qvm-open-in-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes-rpc/qvm-open-in-vm b/qubes-rpc/qvm-open-in-vm index 1461a55..c96d129 100755 --- a/qubes-rpc/qvm-open-in-vm +++ b/qubes-rpc/qvm-open-in-vm @@ -24,6 +24,6 @@ if ! [ $# = 2 ] ; then echo "Usage: $0 vmname filename" exit 1 fi -. /usr/lib/qubes/wrap_in_html_if_url.sh +. /usr/lib/qubes/wrap-in-html-if-url.sh wrap_in_html_if_url "$2" exec /usr/lib/qubes/qrexec-client-vm "$1" qubes.OpenInVM "/usr/lib/qubes/qopen-in-vm" "$FILE_ARGUMENT" From 8f840e10dccc340b721f305ed19130852f75041e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 14 Nov 2013 21:38:27 +0100 Subject: [PATCH 17/32] vm-file-editor: add override for mimeinfo *.png entry (#753) MIME-info database contains multiple entries for *.png, namely image/png and image/x-apple-ios-png. The later one doesn't have associated handler program, but this one is selected by mimeopen tool. Not sure how this tool should behave in case of multiple matches (IOW is it a bug in File::MimeInfo perl module used by mimeopen). Instead of switching to different tool, which probably will break other files (check #423), add override for this particular file type. --- Makefile | 2 ++ misc/mime-globs | 1 + qubes-rpc/vm-file-editor.c | 8 ++++++-- rpm_spec/core-vm.spec | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 misc/mime-globs diff --git a/Makefile b/Makefile index ac0d382..31a7b4b 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,8 @@ install-vm: install -D -m 0644 misc/qubes-trigger-sync-appmenus.action $(DESTDIR)/etc/yum/post-actions/qubes-trigger-sync-appmenus.action install -D misc/polkit-1-qubes-allow-all.pkla $(DESTDIR)/etc/polkit-1/localauthority/50-local.d/qubes-allow-all.pkla install -D misc/polkit-1-qubes-allow-all.rules $(DESTDIR)/etc/polkit-1/rules.d/00-qubes-allow-all.rules + install -D -m 0644 misc/mime-globs $(DESTDIR)/usr/share/qubes/mime-override/globs + mkdir -p $(DESTDIR)/usr/lib/qubes if [ -r misc/dispvm-dotfiles.$(DIST).tbz ] ; \ diff --git a/misc/mime-globs b/misc/mime-globs new file mode 100644 index 0000000..2bc7b9d --- /dev/null +++ b/misc/mime-globs @@ -0,0 +1 @@ +image/png:*.png diff --git a/qubes-rpc/vm-file-editor.c b/qubes-rpc/vm-file-editor.c index a4e53df..657d87c 100644 --- a/qubes-rpc/vm-file-editor.c +++ b/qubes-rpc/vm-file-editor.c @@ -8,6 +8,9 @@ #include #include "dvm2.h" +#define USER_HOME "/home/user" +#define MIMEINFO_DATABASES "/usr/share/mime:/usr/local/share:" USER_HOME "/.local/share:/usr/share/qubes/mime-override" + char *gettime() { static char retbuf[60]; @@ -130,9 +133,10 @@ main() dup2(log_fd, 1); close(log_fd); - setenv("HOME", "/home/user", 1); + setenv("HOME", USER_HOME, 1); setenv("DISPLAY", ":0", 1); - execl("/usr/bin/mimeopen", "mimeopen", "-n", filename, (char*)NULL); + execl("/usr/bin/mimeopen", "mimeopen", "-n", + "--database", MIMEINFO_DATABASES, filename, (char*)NULL); perror("execl"); exit(1); default: diff --git a/rpm_spec/core-vm.spec b/rpm_spec/core-vm.spec index 1834f66..567f481 100644 --- a/rpm_spec/core-vm.spec +++ b/rpm_spec/core-vm.spec @@ -334,6 +334,8 @@ rm -f %{name}-%{version} /usr/share/glib-2.0/schemas/org.gnome.settings-daemon.plugins.updates.gschema.override /usr/share/file-manager/actions/qvm-copy-gnome.desktop /usr/share/file-manager/actions/qvm-dvm-gnome.desktop +%dir /usr/share/qubes +/usr/share/qubes/mime-override/globs %dir /home_volatile %attr(700,user,user) /home_volatile/user %dir /mnt/removable From fa6bb43c62a9a094db3858d4f79f7fba65515487 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 21:26:31 +0100 Subject: [PATCH 18/32] backup: Use paths sent from dom0 to filter files that should be extracted --- qubes-rpc/qubes.Restore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore index 8480ea4..4befc0d 100644 --- a/qubes-rpc/qubes.Restore +++ b/qubes-rpc/qubes.Restore @@ -1,11 +1,13 @@ echo Starting Restorecopy >2 read args +read paths echo Arguments: $args >2 +echo Paths: $paths >2 if [ -f "$args" ] ; then echo "Performing restore from backup file $args" >2 TARGET="$args" echo "Copying $TARGET to STDOUT" >2 - /usr/lib/qubes/tar2qfile $TARGET + /usr/lib/qubes/tar2qfile $TARGET $paths else echo "Checking if arguments is matching a command" >2 COMMAND=`echo $args | cut -d ' ' -f 1` @@ -20,7 +22,7 @@ else # Use named pipe to pass original stdin to tar2file $@ > $tmpdir/backup-data < /dev/null & retcode=$? - /usr/lib/qubes/tar2qfile $tmpdir/backup-data + /usr/lib/qubes/tar2qfile $tmpdir/backup-data $paths wait rm $tmpdir/backup-data rmdir $tmpdir From 91b84d863c4ef1c5ceb51be208ca4382863cdf01 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 21:12:27 +0100 Subject: [PATCH 19/32] tar2qfile: add filtering options to tar2qfile --- qubes-rpc/tar2qfile.c | 45 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 408150a..340d97b 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -405,13 +405,15 @@ int n_dirs = 0; char ** dirs_headers_sent = NULL; int -ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb) +ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb, int filter_count, char **filter) { register HD_USTAR *hd; register char *dest; register int cnt = 0; int ret; + int i; + int should_extract; /* DISABLED: unused dev_t devmajor; dev_t devminor; @@ -620,7 +622,23 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s fprintf(stderr,"File is AREGTYPE\n"); break; case REGTYPE: - fprintf(stderr,"File is REGTYPE of size %ld\n",sb->st_size); + fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); + + // Check if user want to extract this file + should_extract = 1; + for (i=1; i < filter_count; i++) { + should_extract = 0; + fprintf(stderr, "Comparing with filter %s\n", filter[i]); + if (strstr(untrusted_namebuf, filter[i]) == untrusted_namebuf) { + fprintf(stderr, "Match\n"); + should_extract = 1; + break; + } + } + if (should_extract != 1) { + fprintf(stderr, "File should be filtered.. Skipping\n"); + return NEED_SKIP; + } // Create a copy of untrusted_namebuf to be used for strtok char * dirbuf; @@ -756,7 +774,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s -void tar_file_processor(int fd) +int tar_file_processor(int fd, int filter_count, char **filter) { int ret; int i; @@ -798,7 +816,7 @@ void tar_file_processor(int fd) fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); - current = NEED_READ; + current = NEED_SYNC_TRAIL; } i++; } @@ -812,7 +830,11 @@ int main(int argc, char **argv) { int i; char *entry; + char *cwd; + char *sep; int fd; + int use_stdin = 0; + signal(SIGPIPE, SIG_IGN); // this will allow checking for possible feedback packet in the middle of transfer @@ -829,6 +851,9 @@ int main(int argc, char **argv) if (strcmp(argv[i], "--ignore-symlinks")==0) { ignore_symlinks = 1; continue; + } else if (strcmp(argv[i], "-")==0) { + use_stdin = 1; + break; } else { // Parse tar file entry = argv[i]; @@ -838,15 +863,21 @@ int main(int argc, char **argv) if (fd < 0) fprintf(stderr,"Error opening file %s\n",entry); - tar_file_processor(fd); + // At least two arguments can be found in the command line + // (process name and the file to extract) + tar_file_processor(fd, argc-2, argv[2]); } } - if (i <= 1) { + if (i <= 1 || use_stdin == 1) { // No argument specified. Use STDIN fprintf(stderr,"Using STDIN\n"); set_block(0); - tar_file_processor(fileno(stdin)); + // If at least one argument has been found ( process name and - ) + if (use_stdin) + tar_file_processor(fileno(stdin), argc-2, argv[2]); + else + tar_file_processor(fileno(stdin), argc-1, argv[1]); } From 8a1f87d0edfa1e353bf51c72cb3774ddc534747b Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 21:15:33 +0100 Subject: [PATCH 20/32] tar2qfile: starting cleanup to get rid of the stat structure which is not required during conversion --- qubes-rpc/tar2qfile.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 340d97b..8ce851a 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -743,8 +743,8 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } } // Extract extra padding - fprintf(stderr,"Need to remove pad:%ld %ld\n",sb->st_size,BLKMULT-(sb->st_size%BLKMULT)); - ret = read(fd, buf, BLKMULT-(sb->st_size%BLKMULT)); + fprintf(stderr,"Need to remove pad:%ld %ld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); + ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); // Resync trailing headers in order to find next file chunck in the tar file @@ -806,15 +806,15 @@ int tar_file_processor(int fd, int filter_count, char **filter) fprintf(stderr,"Return %d\n",ret); } if (current==NEED_SKIP) { - fprintf(stderr,"Need to skip %ld bytes\n",sb.st_size); - to_skip = sb.st_size; + fprintf(stderr,"Need to skip %ld bytes\n",hdr.filelen); + to_skip = hdr.filelen; while (to_skip > 0) { to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); } // Extract extra padding - fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,sb.st_size,BLKMULT-(sb.st_size%BLKMULT)); - ret = read(fd, &buf, BLKMULT-(sb.st_size%BLKMULT)); + fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); + ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); current = NEED_SYNC_TRAIL; } From 8e853c752c3f69547da26e10a8b6dfbb5ffafa3a Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 21:18:18 +0100 Subject: [PATCH 21/32] tar2qfile: fixed a bug when file contained in tar is a multiple of 512 bytes --- qubes-rpc/tar2qfile.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 8ce851a..d7a9a97 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -744,7 +744,8 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } // Extract extra padding fprintf(stderr,"Need to remove pad:%ld %ld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); - ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); + if (untrusted_hdr->filelen%BLKMULT < BLKMULT) + ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); // Resync trailing headers in order to find next file chunck in the tar file @@ -814,8 +815,10 @@ int tar_file_processor(int fd, int filter_count, char **filter) // Extract extra padding fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); - ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); - fprintf(stderr,"Removed %d bytes of padding\n",ret); + if (hdr.filelen%BLKMULT < BLKMULT) { + ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); + fprintf(stderr,"Removed %d bytes of padding\n",ret); + } current = NEED_SYNC_TRAIL; } i++; From ae776521b090a54c305dc8719ad5f874c77c77db Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 21:21:06 +0100 Subject: [PATCH 22/32] tar2qfile: forgot to pass a parameter in tar_read function --- qubes-rpc/tar2qfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index d7a9a97..cd47d4f 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -803,7 +803,7 @@ int tar_file_processor(int fd, int filter_count, char **filter) } } if (current==NEED_READ) { - current = ustar_rd(fd, &hdr, buf, &sb); + current = ustar_rd(fd, &hdr, buf, &sb, filter_count, filter); fprintf(stderr,"Return %d\n",ret); } if (current==NEED_SKIP) { From 42c40d399b76f08e0c99af7de3077d2aecc50845 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 19 Nov 2013 13:35:51 +0100 Subject: [PATCH 23/32] restore: improve error handling --- qubes-rpc/qubes.Restore | 19 ++++++++++--------- qubes-rpc/tar2qfile.c | 13 ++++++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore index 4befc0d..d1f3303 100644 --- a/qubes-rpc/qubes.Restore +++ b/qubes-rpc/qubes.Restore @@ -1,21 +1,21 @@ -echo Starting Restorecopy >2 +echo Starting Restorecopy >&2 read args read paths -echo Arguments: $args >2 -echo Paths: $paths >2 +echo Arguments: $args >&2 +echo Paths: $paths >&2 if [ -f "$args" ] ; then - echo "Performing restore from backup file $args" >2 + echo "Performing restore from backup file $args" >&2 TARGET="$args" - echo "Copying $TARGET to STDOUT" >2 + echo "Copying $TARGET to STDOUT" >&2 /usr/lib/qubes/tar2qfile $TARGET $paths else - echo "Checking if arguments is matching a command" >2 + echo "Checking if arguments is matching a command" >&2 COMMAND=`echo $args | cut -d ' ' -f 1` TYPE=`type -t $COMMAND` if [ "$TYPE" == "file" ] ; then tmpdir=`mktemp -d` mkfifo $tmpdir/backup-data - echo "Redirecting $args to STDOUT" >2 + echo "Redirecting $args to STDOUT" >&2 # Parsing args to handle quotes correctly # Dangerous method if args are uncontrolled eval "set -- $args" @@ -28,7 +28,8 @@ else rmdir $tmpdir exit $retcode else - echo "Invalid command $COMMAND" >2 - exit 1 + echo "Invalid command $COMMAND" >&2 + exit 2 fi fi + diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index cd47d4f..cd27f3e 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -863,24 +863,27 @@ int main(int argc, char **argv) fprintf(stderr,"Parsing file %s\n",entry); fd = open(entry, O_RDONLY); - if (fd < 0) + if (fd < 0) { fprintf(stderr,"Error opening file %s\n",entry); + exit(2); + } // At least two arguments can be found in the command line // (process name and the file to extract) - tar_file_processor(fd, argc-2, argv[2]); + tar_file_processor(fd, argc, argv); + break; } } - if (i <= 1 || use_stdin == 1) { + if (use_stdin == 1) { // No argument specified. Use STDIN fprintf(stderr,"Using STDIN\n"); set_block(0); // If at least one argument has been found ( process name and - ) if (use_stdin) - tar_file_processor(fileno(stdin), argc-2, argv[2]); + tar_file_processor(fileno(stdin), argc, argv); else - tar_file_processor(fileno(stdin), argc-1, argv[1]); + tar_file_processor(fileno(stdin), argc, argv); } From 3c43f20d9e94bc75180c7ae9b373cbc3b75e9215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:35:03 +0100 Subject: [PATCH 24/32] tar2qfile: terminate parsing when all requested files/dirs found Assume that all the files of directory are in continuous block (which is true in case of qvm-backup stream). This will allow to terminate before getting to the file end - especially useful when only qubes.xml requested. --- qubes-rpc/tar2qfile.c | 76 +++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index cd27f3e..34be38e 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -168,6 +168,13 @@ char *gnu_hack_string; /* GNU ././@LongLink hackery */ char untrusted_namebuf[MAX_PATH_LENGTH]; extern int ignore_quota_error; +struct filters { + int filters_count; + char **filters; + int *filters_matches; + int matched_filters; +}; + /* * asc_ul() @@ -386,6 +393,7 @@ enum { enum { NEED_NOTHING, NEED_SKIP, + NEED_SKIP_FILE, // distinguish between skipped file and unwanted blocks (extended headers etc) NEED_READ, NEED_SYNC_TRAIL, INVALID_HEADER, @@ -405,7 +413,7 @@ int n_dirs = 0; char ** dirs_headers_sent = NULL; int -ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb, int filter_count, char **filter) +ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * sb, struct filters *filters) { register HD_USTAR *hd; @@ -626,18 +634,23 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // Check if user want to extract this file should_extract = 1; - for (i=1; i < filter_count; i++) { + for (i=0; i < filters->filters_count; i++) { should_extract = 0; - fprintf(stderr, "Comparing with filter %s\n", filter[i]); - if (strstr(untrusted_namebuf, filter[i]) == untrusted_namebuf) { - fprintf(stderr, "Match\n"); + fprintf(stderr, "Comparing with filter %s\n", filters->filters[i]); + if (strncmp(untrusted_namebuf, filters->filters[i], strlen(filters->filters[i])) == 0) { + fprintf(stderr, "Match (%d)\n", filters->filters_matches[i]); should_extract = 1; + filters->filters_matches[i]++; + if (filters->filters_matches[i] == 1) { + // first match + filters->matched_filters++; + } break; } } if (should_extract != 1) { fprintf(stderr, "File should be filtered.. Skipping\n"); - return NEED_SKIP; + return NEED_SKIP_FILE; } // Create a copy of untrusted_namebuf to be used for strtok @@ -775,7 +788,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s -int tar_file_processor(int fd, int filter_count, char **filter) +void tar_file_processor(int fd, struct filters *filters) { int ret; int i; @@ -803,11 +816,28 @@ int tar_file_processor(int fd, int filter_count, char **filter) } } if (current==NEED_READ) { - current = ustar_rd(fd, &hdr, buf, &sb, filter_count, filter); + current = ustar_rd(fd, &hdr, buf, &sb, filters); fprintf(stderr,"Return %d\n",ret); } - if (current==NEED_SKIP) { - fprintf(stderr,"Need to skip %ld bytes\n",hdr.filelen); + if (current==NEED_SKIP || current==NEED_SKIP_FILE) { + if (current==NEED_SKIP_FILE && + filters->filters_count > 0 && + filters->filters_count == filters->matched_filters) { + // This assume that either: + // a) files are sorted (using full path as sort key) + // b) all the directory content is in + // consecutive block and only directories + // are given as filters + // This is true for backups prepared by qvm-backup +#ifdef DEBUG + fprintf(stderr, "All filters matched at least once - assuming end of requested data\n"); +#endif + return; + } +#ifdef DEBUG + fprintf(stderr,"Need to skip %lld bytes (matched filters %d < %d)\n", + hdr.filelen, filters->matched_filters, filters->filters_count); +#endif to_skip = hdr.filelen; while (to_skip > 0) { to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); @@ -836,8 +866,8 @@ int main(int argc, char **argv) char *cwd; char *sep; int fd; - int use_stdin = 0; - + int use_stdin = 1; + struct filters filters; signal(SIGPIPE, SIG_IGN); // this will allow checking for possible feedback packet in the middle of transfer @@ -856,9 +886,11 @@ int main(int argc, char **argv) continue; } else if (strcmp(argv[i], "-")==0) { use_stdin = 1; + i++; break; } else { // Parse tar file + use_stdin = 0; entry = argv[i]; fprintf(stderr,"Parsing file %s\n",entry); @@ -867,24 +899,26 @@ int main(int argc, char **argv) fprintf(stderr,"Error opening file %s\n",entry); exit(2); } - - // At least two arguments can be found in the command line - // (process name and the file to extract) - tar_file_processor(fd, argc, argv); + i++; break; } } + filters.filters_count = argc-i; + filters.filters = argv+i; + filters.filters_matches = calloc(filters.filters_count, sizeof(int)); + if (filters.filters_matches == NULL) { + perror("calloc"); + exit(1); + } + filters.matched_filters = 0; if (use_stdin == 1) { // No argument specified. Use STDIN fprintf(stderr,"Using STDIN\n"); set_block(0); - // If at least one argument has been found ( process name and - ) - if (use_stdin) - tar_file_processor(fileno(stdin), argc, argv); - else - tar_file_processor(fileno(stdin), argc, argv); + fd = 0; } + tar_file_processor(fd, &filters); //notify_end_and_wait_for_result(); From 52d696a0c3359aba88f7eb85cbb06ddddf56cf97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:38:36 +0100 Subject: [PATCH 25/32] tar2qfile: fix padding handling --- qubes-rpc/tar2qfile.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 34be38e..afd1da8 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -756,8 +756,8 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } } // Extract extra padding - fprintf(stderr,"Need to remove pad:%ld %ld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); - if (untrusted_hdr->filelen%BLKMULT < BLKMULT) + fprintf(stderr,"Need to remove pad:%lld %lld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); + if (untrusted_hdr->filelen%BLKMULT > 0) ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); @@ -844,8 +844,8 @@ void tar_file_processor(int fd, struct filters *filters) } // Extract extra padding - fprintf(stderr,"Need to remove pad:%ld %ld %ld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); - if (hdr.filelen%BLKMULT < BLKMULT) { + fprintf(stderr,"Need to remove pad:%ld %lld %lld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); + if (hdr.filelen%BLKMULT > 0) { ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); fprintf(stderr,"Removed %d bytes of padding\n",ret); } From 584df6986e20fcf8664d49e1742616f4cb0f1726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:39:36 +0100 Subject: [PATCH 26/32] tar2qfile: fix compile warnings --- qubes-rpc/tar2qfile.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index afd1da8..d3fe3fe 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -630,7 +630,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s fprintf(stderr,"File is AREGTYPE\n"); break; case REGTYPE: - fprintf(stderr,"File is REGTYPE of size %d\n",sb->st_size); + fprintf(stderr,"File is REGTYPE of size %ld\n",sb->st_size); // Check if user want to extract this file should_extract = 1; @@ -863,8 +863,6 @@ int main(int argc, char **argv) { int i; char *entry; - char *cwd; - char *sep; int fd; int use_stdin = 1; struct filters filters; From dba35718836c8cc5ed008583d6612b61204862a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:41:00 +0100 Subject: [PATCH 27/32] tar2qfile: send EOF marker --- qubes-rpc/tar2qfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index d3fe3fe..3849583 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -919,7 +919,7 @@ int main(int argc, char **argv) tar_file_processor(fd, &filters); - //notify_end_and_wait_for_result(); + notify_end_and_wait_for_result(); notify_progress(0, PROGRESS_FLAG_DONE); return 0; } From 03923ae5489683eb23c3a3fdc1bbb25605aa3cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:42:12 +0100 Subject: [PATCH 28/32] tar2qfile: disable debug messages --- qubes-rpc/tar2qfile.c | 89 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 3849583..e7ec11f 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -48,6 +48,8 @@ #include #include +// #define DEBUG + /*************************************************** * Most routines extracted from the PAX project (tar.c...) * ***************************************************/ @@ -430,12 +432,18 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s /* * we only get proper sized buffers */ +#ifdef DEBUG fprintf(stderr,"Checking if valid header\n"); +#endif if (ustar_id (buf, BLKMULT) < 0) { - fprintf (stderr,"Invalid header\n"); +#ifdef DEBUG + fprintf (stderr, "Invalid header\n"); +#endif return INVALID_HEADER; } +#ifdef DEBUG fprintf(stderr,"Valid header!\n"); +#endif /* DISABLED: Internal to PAX arcn->org_name = arcn->name; arcn->sb.st_nlink = 1; @@ -472,8 +480,10 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // qfile count the \0 in the namelen untrusted_hdr->namelen += 1; +#ifdef DEBUG fprintf(stderr,"Retrieved name len: %d\n",untrusted_hdr->namelen); fprintf(stderr,"Retrieved name: %s\n",untrusted_namebuf); +#endif /* * follow the spec to the letter. we should only have mode bits, strip @@ -523,7 +533,7 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s arcn->skip = 0; arcn->sb.st_rdev = (dev_t) 0; */ - + /* * set the mode and PAX type according to the typeflag in the header @@ -531,14 +541,18 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s switch (hd->typeflag) { case FIFOTYPE: +#ifdef DEBUG fprintf(stderr,"File is FIFOTYPE\n"); +#endif /* DISABLED: unused arcn->type = PAX_FIF; arcn->sb.st_mode |= S_IFIFO; */ break; case DIRTYPE: +#ifdef DEBUG fprintf(stderr,"File is DIRTYPE\n"); +#endif /* DISABLED: unused arcn->type = PAX_DIR; arcn->sb.st_mode |= S_IFDIR; @@ -555,10 +569,14 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s */ break; case BLKTYPE: +#ifdef DEBUG fprintf(stderr,"File is BLKTYPE\n"); +#endif break; case CHRTYPE: +#ifdef DEBUG fprintf(stderr,"File is CHRTYPE\n"); +#endif /* * this type requires the rdev field to be set. */ @@ -583,10 +601,14 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // arcn->sb.st_rdev = TODEV (devmajor, devminor); break; case SYMTYPE: +#ifdef DEBUG fprintf(stderr,"File is SYMTYPE\n"); +#endif break; case LNKTYPE: +#ifdef DEBUG fprintf(stderr,"File is LNKTYPE\n"); +#endif if (hd->typeflag == SYMTYPE) { // arcn->type = PAX_SLK; @@ -608,10 +630,14 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // MIN(TNMSZ+1,sizeof (arcn->ln_name))); break; case LONGLINKTYPE: +#ifdef DEBUG fprintf(stderr,"File is LONGLINKTYPE\n"); +#endif break; case LONGNAMETYPE: +#ifdef DEBUG fprintf(stderr,"File is LONGNAMETYPE\n"); +#endif /* * GNU long link/file; we tag these here and let the * pax internals deal with it -- too ugly otherwise. @@ -624,21 +650,31 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s // arcn->ln_nlen = 0; break; case CONTTYPE: +#ifdef DEBUG fprintf(stderr,"File is CONTTYPE\n"); +#endif break; case AREGTYPE: +#ifdef DEBUG fprintf(stderr,"File is AREGTYPE\n"); +#endif break; case REGTYPE: +#ifdef DEBUG fprintf(stderr,"File is REGTYPE of size %ld\n",sb->st_size); +#endif // Check if user want to extract this file should_extract = 1; for (i=0; i < filters->filters_count; i++) { should_extract = 0; +#ifdef DEBUG fprintf(stderr, "Comparing with filter %s\n", filters->filters[i]); +#endif if (strncmp(untrusted_namebuf, filters->filters[i], strlen(filters->filters[i])) == 0) { +#ifdef DEBUG fprintf(stderr, "Match (%d)\n", filters->filters_matches[i]); +#endif should_extract = 1; filters->filters_matches[i]++; if (filters->filters_matches[i] == 1) { @@ -649,7 +685,9 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } } if (should_extract != 1) { +#ifdef DEBUG fprintf(stderr, "File should be filtered.. Skipping\n"); +#endif return NEED_SKIP_FILE; } @@ -671,7 +709,9 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s char * token = strtok(NULL, "/"); while (token != NULL) { +#ifdef DEBUG fprintf(stderr,"Found directory %s (last:%s)\n",token,last_token); +#endif // Recompose the path based on last discovered directory if (path == NULL) { @@ -690,21 +730,31 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s strncpy(path+pathsize+1, last_token, strlen(last_token)); path[pathsize+strlen(last_token)+1] = '\0'; } +#ifdef DEBUG fprintf(stderr,"Path is %s\n",path); +#endif +#ifdef DEBUG fprintf(stderr,"Checking from i=0 i<%d\n",n_dirs); +#endif // Verify if qfile headers for the current path have already been sent based on the dirs_headers_sent table dir_found = 0; for (i = 0; i < n_dirs; ++i) { +#ifdef DEBUG fprintf(stderr,"Comparing with %d %d %s %s\n",i,n_dirs,dirs_headers_sent[i],path); +#endif if (strcmp(dirs_headers_sent[i],path)==0) { +#ifdef DEBUG fprintf(stderr,"Directory headers already sent\n"); +#endif dir_found=1; } } if (dir_found == 0) { // Register the current path as being sent in the dirs_headers_sent table +#ifdef DEBUG fprintf(stderr,"Inserting %s into register\n",path); +#endif dirs_headers_sent = realloc(dirs_headers_sent, sizeof (char*) * n_dirs++); if (dirs_headers_sent == NULL) return MEMORY_ALLOC_FAILED; @@ -723,7 +773,9 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s dir_header.mode = untrusted_hdr->mode | S_IFDIR; dir_header.filelen = 0; +#ifdef DEBUG fprintf(stderr,"Sending directory headers for %s\n",path); +#endif // Send the qfile headers for the current directory path write_headers(&dir_header, path); } @@ -733,17 +785,25 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s free(path); free(dirbuf); +#ifdef DEBUG fprintf(stderr,"End of directory checks\n"); +#endif // Restore POSIX stat file mode (because PAX format use its own file type) untrusted_hdr->mode |= S_IFREG; +#ifdef DEBUG fprintf(stderr,"Writing file header\n"); +#endif // Send header and file content write_headers(untrusted_hdr, untrusted_namebuf); +#ifdef DEBUG fprintf(stderr,"Writing file content\n"); +#endif ret = copy_file(1, fd, untrusted_hdr->filelen, &crc32_sum); +#ifdef DEBUG fprintf(stderr,"Copyfile returned with error %d\n",ret); +#endif if (ret != COPY_FILE_OK) { if (ret != COPY_FILE_WRITE_ERROR) gui_fatal("Copying file %s: %s", untrusted_namebuf, @@ -756,22 +816,30 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s } } // Extract extra padding +#ifdef DEBUG fprintf(stderr,"Need to remove pad:%lld %lld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); +#endif if (untrusted_hdr->filelen%BLKMULT > 0) ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); +#ifdef DEBUG fprintf(stderr,"Removed %d bytes of padding\n",ret); +#endif // Resync trailing headers in order to find next file chunck in the tar file return NEED_SYNC_TRAIL; break; case EXTHEADERTYPE: +#ifdef DEBUG fprintf(stderr,"Extended HEADER encountered\n"); +#endif return NEED_SKIP; break; default: +#ifdef DEBUG fprintf(stderr,"Default type detected:%c\n",hd->typeflag); +#endif return NEED_SKIP; /* * these types have file data that follows. Set the skip and @@ -806,18 +874,24 @@ void tar_file_processor(int fd, struct filters *filters) int sync_count = 0; while ((size = read(fd, &buf, BLKMULT))) { if (size != -1) { +#ifdef DEBUG fprintf(stderr,"Read %ld bytes\n",size); +#endif ret = 0; if (current==NEED_SYNC_TRAIL) { ret = tar_trail (buf, 1, &sync_count); +#ifdef DEBUG fprintf(stderr,"Synchronizing trail: %d %d\n",ret,sync_count); +#endif if (ret != 1) { current = NEED_READ; } } if (current==NEED_READ) { current = ustar_rd(fd, &hdr, buf, &sb, filters); +#ifdef DEBUG fprintf(stderr,"Return %d\n",ret); +#endif } if (current==NEED_SKIP || current==NEED_SKIP_FILE) { if (current==NEED_SKIP_FILE && @@ -844,13 +918,17 @@ void tar_file_processor(int fd, struct filters *filters) } // Extract extra padding +#ifdef DEBUG fprintf(stderr,"Need to remove pad:%ld %lld %lld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); +#endif if (hdr.filelen%BLKMULT > 0) { ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); +#ifdef DEBUG fprintf(stderr,"Removed %d bytes of padding\n",ret); +#endif } current = NEED_SYNC_TRAIL; - } + } i++; } //if (i >= 10) @@ -890,7 +968,9 @@ int main(int argc, char **argv) // Parse tar file use_stdin = 0; entry = argv[i]; +#ifdef DEBUG fprintf(stderr,"Parsing file %s\n",entry); +#endif fd = open(entry, O_RDONLY); if (fd < 0) { @@ -911,8 +991,9 @@ int main(int argc, char **argv) filters.matched_filters = 0; if (use_stdin == 1) { - // No argument specified. Use STDIN +#ifdef DEBUG fprintf(stderr,"Using STDIN\n"); +#endif set_block(0); fd = 0; } From 4010ddaab58d46a009dfa3cc31cf5e1dcee5ac0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:45:36 +0100 Subject: [PATCH 29/32] minor whitespace fix --- qubes-rpc/qubes.Restore | 1 - 1 file changed, 1 deletion(-) diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore index d1f3303..c77774f 100644 --- a/qubes-rpc/qubes.Restore +++ b/qubes-rpc/qubes.Restore @@ -32,4 +32,3 @@ else exit 2 fi fi - From 9b859c9ac55af6d533cb50149f7a120fc9792304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 02:09:36 +0100 Subject: [PATCH 30/32] qubes-rpc: save one syscall on each data block read_all/write_all calls set_* on every call, so this can be noticeable performance improvement. --- qubes-rpc/ioall.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qubes-rpc/ioall.c b/qubes-rpc/ioall.c index 2a81df4..ce61700 100644 --- a/qubes-rpc/ioall.c +++ b/qubes-rpc/ioall.c @@ -35,12 +35,16 @@ void perror_wrapper(char * msg) void set_nonblock(int fd) { int fl = fcntl(fd, F_GETFL, 0); + if (fl & O_NONBLOCK) + return; fcntl(fd, F_SETFL, fl | O_NONBLOCK); } void set_block(int fd) { int fl = fcntl(fd, F_GETFL, 0); + if (!(fl & O_NONBLOCK)) + return; fcntl(fd, F_SETFL, fl & ~O_NONBLOCK); } From 3c92cdba22a08706945df10fc7b89fd9255e1bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 02:24:21 +0100 Subject: [PATCH 31/32] tar2qfile: use read_all() instead of read() read() syscall do not guarantee to read as much data as requested. This is especially important when reading from pipe - remote end can produce data slower than we are reading them. Use read_all() helper to always get requested amount of data. --- qubes-rpc/tar2qfile.c | 116 ++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index e7ec11f..516eaaa 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -819,11 +819,12 @@ ustar_rd (int fd, struct file_header * untrusted_hdr, char *buf, struct stat * s #ifdef DEBUG fprintf(stderr,"Need to remove pad:%lld %lld\n",untrusted_hdr->filelen,BLKMULT-(untrusted_hdr->filelen%BLKMULT)); #endif - if (untrusted_hdr->filelen%BLKMULT > 0) - ret = read(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT)); -#ifdef DEBUG - fprintf(stderr,"Removed %d bytes of padding\n",ret); -#endif + if (untrusted_hdr->filelen%BLKMULT > 0) { + if (!read_all(fd, buf, BLKMULT-(untrusted_hdr->filelen%BLKMULT))) { + wait_for_result(); + exit(1); + } + } // Resync trailing headers in order to find next file chunck in the tar file return NEED_SYNC_TRAIL; @@ -866,71 +867,64 @@ void tar_file_processor(int fd, struct filters *filters) struct stat sb; /* stat buffer see stat(2) */ char buf[BLKMULT+1]; - size_t size; i=0; current = NEED_READ; size_t to_skip = 0; int sync_count = 0; - while ((size = read(fd, &buf, BLKMULT))) { - if (size != -1) { + while (read_all(fd, buf, BLKMULT)) { + ret = 0; + if (current==NEED_SYNC_TRAIL) { + ret = tar_trail (buf, 1, &sync_count); #ifdef DEBUG - fprintf(stderr,"Read %ld bytes\n",size); + fprintf(stderr,"Synchronizing trail: %d %d\n", ret, sync_count); #endif - ret = 0; - if (current==NEED_SYNC_TRAIL) { - ret = tar_trail (buf, 1, &sync_count); -#ifdef DEBUG - fprintf(stderr,"Synchronizing trail: %d %d\n",ret,sync_count); -#endif - if (ret != 1) { - current = NEED_READ; - } + if (ret != 1) { + current = NEED_READ; + sync_count = 0; } - if (current==NEED_READ) { - current = ustar_rd(fd, &hdr, buf, &sb, filters); -#ifdef DEBUG - fprintf(stderr,"Return %d\n",ret); -#endif - } - if (current==NEED_SKIP || current==NEED_SKIP_FILE) { - if (current==NEED_SKIP_FILE && - filters->filters_count > 0 && - filters->filters_count == filters->matched_filters) { - // This assume that either: - // a) files are sorted (using full path as sort key) - // b) all the directory content is in - // consecutive block and only directories - // are given as filters - // This is true for backups prepared by qvm-backup -#ifdef DEBUG - fprintf(stderr, "All filters matched at least once - assuming end of requested data\n"); -#endif - return; - } -#ifdef DEBUG - fprintf(stderr,"Need to skip %lld bytes (matched filters %d < %d)\n", - hdr.filelen, filters->matched_filters, filters->filters_count); -#endif - to_skip = hdr.filelen; - while (to_skip > 0) { - to_skip -= read(fd, &buf, MIN(to_skip,BLKMULT)); - } - - // Extract extra padding -#ifdef DEBUG - fprintf(stderr,"Need to remove pad:%ld %lld %lld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); -#endif - if (hdr.filelen%BLKMULT > 0) { - ret = read(fd, &buf, BLKMULT-(hdr.filelen%BLKMULT)); -#ifdef DEBUG - fprintf(stderr,"Removed %d bytes of padding\n",ret); -#endif - } - current = NEED_SYNC_TRAIL; - } - i++; } + if (current==NEED_READ) { + current = ustar_rd(fd, &hdr, buf, &sb, filters); +#ifdef DEBUG + fprintf(stderr,"Return %d\n", current); +#endif + } + if (current==NEED_SKIP || current==NEED_SKIP_FILE) { + if (current==NEED_SKIP_FILE && + filters->filters_count > 0 && + filters->filters_count == filters->matched_filters) { + // This assume that either: + // a) files are sorted (using full path as sort key) + // b) all the directory content is in + // consecutive block and only directories + // are given as filters + // This is true for backups prepared by qvm-backup +#ifdef DEBUG + fprintf(stderr, "All filters matched at least once - assuming end of requested data\n"); +#endif + return; + } + to_skip = hdr.filelen; +#ifdef DEBUG + fprintf(stderr,"Need to skip %lld bytes (matched filters %d < %d)\n", + hdr.filelen, filters->matched_filters, filters->filters_count); + fprintf(stderr,"Need to remove pad:%ld %lld %lld\n",to_skip,hdr.filelen,BLKMULT-(hdr.filelen%BLKMULT)); +#endif + if (to_skip%BLKMULT > 0) { + to_skip += BLKMULT-(to_skip%BLKMULT); + } + while (to_skip > 0) { + ret = read_all(fd, &buf, MIN(to_skip,BLKMULT)); + if (ret <= 0) { + exit(1); + } + to_skip -= MIN(to_skip,BLKMULT); + } + + current = NEED_SYNC_TRAIL; + } + i++; //if (i >= 10) // exit(0); } From 6c3c3e717df8875c526e2161d2ed26cce7ece762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 02:28:35 +0100 Subject: [PATCH 32/32] tar2qfile: use lseek() to skip unwanted data if possible When reading from file it is much faster. --- qubes-rpc/tar2qfile.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/qubes-rpc/tar2qfile.c b/qubes-rpc/tar2qfile.c index 516eaaa..eeccc0b 100644 --- a/qubes-rpc/tar2qfile.c +++ b/qubes-rpc/tar2qfile.c @@ -168,6 +168,7 @@ static unsigned long tar_chksm (char *, int); char *gnu_hack_string; /* GNU ././@LongLink hackery */ char untrusted_namebuf[MAX_PATH_LENGTH]; +int use_seek = 1; extern int ignore_quota_error; struct filters { @@ -914,12 +915,27 @@ void tar_file_processor(int fd, struct filters *filters) if (to_skip%BLKMULT > 0) { to_skip += BLKMULT-(to_skip%BLKMULT); } - while (to_skip > 0) { - ret = read_all(fd, &buf, MIN(to_skip,BLKMULT)); - if (ret <= 0) { - exit(1); + if (use_seek) { + ret = lseek(fd, to_skip, SEEK_CUR); + if (ret < 0) { + if (errno == ESPIPE) { + // fallback to read() + use_seek = 0; + } else { + perror("lseek"); + exit(1); + } + } + } + // not using "else" because above can fallback to read() method + if (!use_seek) { + while (to_skip > 0) { + ret = read_all(fd, &buf, MIN(to_skip,BLKMULT)); + if (ret <= 0) { + exit(1); + } + to_skip -= MIN(to_skip,BLKMULT); } - to_skip -= MIN(to_skip,BLKMULT); } current = NEED_SYNC_TRAIL;