From 682138a3544a2d7de457c88712e738938568f908 Mon Sep 17 00:00:00 2001 From: Valentin Samir Date: Wed, 19 Feb 2020 16:40:03 +0100 Subject: [PATCH 1/4] bpo-39688: tarfile: compute next header offset using pax size for sparse file In case of a sparse file, the tarinfo.size attribute is set to the sparse file expanded size (pax attribute GNU.sparse.size or GNU.sparse.size) and do not correspond to the actual size of the data block. The size of the data block is specified by the size pax header if present or by the ustar size header. Moreover, for GNU sparse 1.0 files, the data block start at the beginning of the sparse mapping and not after the sparse mapping and so the offset should be computed from here. --- Lib/tarfile.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index e2b60532f693d4..60c4eb97bb6860 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1292,17 +1292,22 @@ def _proc_pax(self, tarfile): if self.type in (XHDTYPE, SOLARIS_XHDTYPE): # Patch the TarInfo object with the extended header info. next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset if "size" in pax_headers: # If the extended header replaces the size field, # we need to recalculate the offset where the next # header starts. - offset = next.offset_data + offset = next.offset + BLOCKSIZE if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) + try: + size = PAX_NUMBER_FIELDS["size"](pax_headers["size"]) + except ValueError: + size = 0 + offset += next._block(size) tarfile.offset = offset + next.offset = self.offset + return next def _proc_gnusparse_00(self, next, pax_headers, buf): From e3fb592e79a01098d5d7f2b154773bb6aaa8a791 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2020 16:35:54 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst diff --git a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst new file mode 100644 index 00000000000000..5ac73115e562fe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst @@ -0,0 +1,2 @@ +Compute next header offset using pax size for sparse file instead of the +sparse real size (expended size). \ No newline at end of file From 1ba542ddf05eb73ad46e32a91c2308ce41fb7cfe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 14 Aug 2026 17:54:13 +0300 Subject: [PATCH 3/4] Add a test and rename the NEWS entry The test builds a small archive with a pax size header for a GNU sparse 1.0 member, so that it does not need a file larger than 8 GiB. GNU tar, bsdtar and 7-Zip all read such an archive and find the member after the sparse file. --- Lib/test/test_tarfile.py | 31 +++++++++++++++++++ .../2020-02-19-16-35-52.bpo-39688.EPD_zn.rst | 2 -- ...0-02-19-16-35-52.gh-issue-83869.EPD_zn.rst | 6 ++++ 3 files changed, 37 insertions(+), 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst create mode 100644 Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 5fa97e2ac226c4..ed7a64804fdf12 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1399,6 +1399,37 @@ def test_sparse_file_01(self): def test_sparse_file_10(self): self._test_sparse_file("gnu/sparse-1.0") + def test_sparse_file_10_pax_size(self): + # gh-83869: when the pax header replaces the size field, the offset + # of the next header must be computed from the size of the data in + # the archive, not from the apparent size of the sparse file. + data = b"payload!" * 4 + realsize = 1 << 20 + smap = b"1\n%d\n%d\n" % (realsize - len(data), len(data)) + smap += b"\0" * (-len(smap) % tarfile.BLOCKSIZE) + + sparse = tarfile.TarInfo("sparse") + sparse.size = len(smap) + len(data) + sparse.pax_headers = { + "GNU.sparse.major": "1", + "GNU.sparse.minor": "0", + "GNU.sparse.name": "sparse", + "GNU.sparse.realsize": str(realsize), + "size": str(sparse.size), + } + buf = sparse.tobuf(tarfile.PAX_FORMAT) + buf += smap + data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + + last = tarfile.TarInfo("last") + last.size = len(data) + buf += last.tobuf(tarfile.PAX_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + self.assertEqual(tar.getnames(), ["sparse", "last"]) + self.assertEqual(tar.extractfile("last").read(), data) + @staticmethod def _fs_supports_holes(): # Return True if the platform knows the st_blocks stat attribute and diff --git a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst deleted file mode 100644 index 5ac73115e562fe..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.bpo-39688.EPD_zn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Compute next header offset using pax size for sparse file instead of the -sparse real size (expended size). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst new file mode 100644 index 00000000000000..efadbb289c3498 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst @@ -0,0 +1,6 @@ +Fix :mod:`tarfile` reading an archive with a GNU sparse 1.0 member whose +size is set in the pax extended header. +The offset of the next header was computed from the offset of the data, +which is already past the sparse map, and from the size of the member, +which can be the apparent size of the sparse file. +All following members were unreachable. From e3d43fdfe0167bacb889a7493af8ce3f1a27ce66 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 14 Aug 2026 18:00:51 +0300 Subject: [PATCH 4/4] Keep the offset of a member which follows a global header _proc_pax() handles the global extended header too, so setting the offset of the next member unconditionally made it the offset of the global header instead of its own. It only has to be set after the offset of the next header is computed. --- Lib/tarfile.py | 2 +- Lib/test/test_tarfile.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index b1f4517d077dc9..c0dd111e557b62 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1625,7 +1625,7 @@ def _proc_pax(self, tarfile): offset += next._block(size) tarfile.offset = offset - next.offset = self.offset + next.offset = self.offset return next diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index ed7a64804fdf12..57d86cfeb5632a 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1503,6 +1503,31 @@ def test_pax_global_headers(self): finally: tar.close() + def test_offset_after_global_header(self): + # gh-83869: a global header is a member of its own, the member which + # follows it keeps the offset of its own header. + rec = b"30 comment=global header here\n" + glob = tarfile.TarInfo("././@PaxHeader") + glob.type = tarfile.XGLTYPE + glob.size = len(rec) + buf = glob.tobuf(tarfile.USTAR_FORMAT) + buf += rec + b"\0" * (-len(rec) % tarfile.BLOCKSIZE) + + member = tarfile.TarInfo("member") + data = b"hello\n" + member.size = len(data) + offset = len(buf) + buf += member.tobuf(tarfile.USTAR_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + tarinfo = tar.getmember("member") + self.assertEqual(tarinfo.offset, offset) + self.assertEqual(tarinfo.pax_headers.get("comment"), + "global header here") + self.assertEqual(tar.extractfile(tarinfo).read(), data) + def test_pax_number_fields(self): # All following number fields are read from the pax header. tar = tarfile.open(tarname, encoding="iso8859-1")