bastion agent 0.1.14
This commit is contained in:
@@ -0,0 +1,541 @@
|
||||
"""The agent's built-in PDF reader (pdftext) on the shapes real writers make.
|
||||
|
||||
The regex reader it replaced returned NOTHING for a browser-printed PDF: text
|
||||
there is glyph codes in hex strings through Type0 / Identity-H fonts, mapped
|
||||
back by /ToUnicode CMaps, one Td per glyph run. These tests build such files
|
||||
with a small writer (no third-party code, no user documents) and read them.
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
import unittest
|
||||
import zlib
|
||||
from unittest import mock
|
||||
|
||||
from . import extract as ex
|
||||
from . import pdftext as pt
|
||||
|
||||
CMAP = b"""/CIDInit /ProcSet findresource begin 12 dict begin begincmap
|
||||
1 begincodespacerange <0000> <FFFF> endcodespacerange
|
||||
2 beginbfchar
|
||||
<0003> <0020>
|
||||
<0010> <00660069>
|
||||
endbfchar
|
||||
1 beginbfrange
|
||||
<0024> <003D> <0041>
|
||||
endbfrange
|
||||
endcmap CMapName currentdict /CMap defineresource pop end end
|
||||
"""
|
||||
|
||||
|
||||
def _enc(text):
|
||||
"""Glyph codes for our CMap: A-Z -> 0x24.., space -> 3, 'fi' -> 0x10."""
|
||||
out = b""
|
||||
i = 0
|
||||
while i < len(text):
|
||||
if text[i:i + 2] == "fi":
|
||||
out += b"\x00\x10"
|
||||
i += 2
|
||||
continue
|
||||
c = text[i]
|
||||
out += b"\x00\x03" if c == " " else (0x24 + ord(c) - 0x41).to_bytes(2, "big")
|
||||
i += 1
|
||||
return b"<" + out.hex().upper().encode() + b">"
|
||||
|
||||
|
||||
class Pdf:
|
||||
def __init__(self):
|
||||
self.objs = []
|
||||
|
||||
def add(self, body, stream=None, compress=True):
|
||||
n = len(self.objs) + 1
|
||||
if stream is not None:
|
||||
data = zlib.compress(stream) if compress else stream
|
||||
flt = b"/Filter /FlateDecode " if compress else b""
|
||||
body = body.rstrip(b">") + b" " + flt + b"/Length " + str(len(data)).encode() + b" >>"
|
||||
self.objs.append(b"%d 0 obj\n" % n + body + b"\nstream\n" + data + b"\nendstream\nendobj\n")
|
||||
else:
|
||||
self.objs.append(b"%d 0 obj\n" % n + body + b"\nendobj\n")
|
||||
return n
|
||||
|
||||
def bytes(self):
|
||||
return b"%PDF-1.7\n" + b"".join(self.objs) + b"trailer << /Root 1 0 R >>\n%%EOF\n"
|
||||
|
||||
|
||||
def browser_like(pages, xobject=False):
|
||||
"""A catalog -> pages tree with RESOURCES ON THE PARENT (inherited), a
|
||||
Type0 Identity-H font with a ToUnicode CMap, and per-glyph-run Td moves."""
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") # 1
|
||||
pdf.add(b"PLACEHOLDER") # 2 pages
|
||||
cmap = pdf.add(b"<< >>", CMAP) # 3
|
||||
font = pdf.add(b"<< /Type /Font /Subtype /Type0 /BaseFont /F /Encoding /Identity-H "
|
||||
b"/ToUnicode %d 0 R >>" % cmap) # 4
|
||||
kids = []
|
||||
form = None
|
||||
if xobject:
|
||||
form = pdf.add(b"<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>",
|
||||
b"BT /F1 9 Tf 1 0 0 1 10 10 Tm " + _enc("FOOTER") + b" Tj ET")
|
||||
for lines in pages:
|
||||
content = b"BT /F1 12 Tf 1 0 0 1 50 700 Tm "
|
||||
y = 700
|
||||
for line in lines:
|
||||
words = line.split(" ")
|
||||
# one Td per word run on the SAME line (x moves only)
|
||||
for k, w in enumerate(words):
|
||||
run = (w + " ") if k < len(words) - 1 else w
|
||||
content += b"[" + _enc(run) + b"] TJ 20 0 Td "
|
||||
y -= 14
|
||||
content += b"1 0 0 1 50 %d Tm " % y
|
||||
content += b"ET"
|
||||
if form:
|
||||
content += b" /X1 Do"
|
||||
c = pdf.add(b"<< >>", content)
|
||||
kids.append(pdf.add(b"<< /Type /Page /Parent 2 0 R /Contents %d 0 R >>" % c))
|
||||
res = b"/Resources << /Font << /F1 %d 0 R >>" % font
|
||||
if form:
|
||||
res += b" /XObject << /X1 %d 0 R >>" % form
|
||||
res += b" >>"
|
||||
pdf.objs[1] = (b"2 0 obj\n<< /Type /Pages /Count %d /Kids [%s] %s >>\nendobj\n"
|
||||
% (len(kids), b" ".join(b"%d 0 R" % k for k in kids), res))
|
||||
return pdf.bytes()
|
||||
|
||||
|
||||
class PdfTextTests(unittest.TestCase):
|
||||
def test_type0_identity_h_with_tounicode(self):
|
||||
data = browser_like([["HELLO WORLD", "SECOND LINE"]])
|
||||
self.assertEqual(pt.extract_text(data), "HELLO WORLD\nSECOND LINE")
|
||||
|
||||
def test_the_old_reader_found_nothing_here(self):
|
||||
# The shape of the regression: no literal (…) strings at all.
|
||||
data = browser_like([["HELLO WORLD"]])
|
||||
for chunk in data.split(b"stream\n")[1:]:
|
||||
raw = chunk.split(b"\nendstream")[0]
|
||||
try:
|
||||
self.assertNotIn(b"HELLO", zlib.decompress(raw))
|
||||
except zlib.error:
|
||||
pass
|
||||
|
||||
def test_words_are_not_shredded_by_per_run_moves(self):
|
||||
out = pt.extract_text(browser_like([["THE QUICK BROWN FOX"]]))
|
||||
self.assertEqual(out, "THE QUICK BROWN FOX")
|
||||
|
||||
def test_ligature_codes_map_back_to_letters(self):
|
||||
out = pt.extract_text(browser_like([["CLARIfiES"]]))
|
||||
self.assertEqual(out, "CLARIFIES".replace("FI", "fi"))
|
||||
|
||||
def test_pages_in_order_and_inherited_resources(self):
|
||||
out = pt.extract_text(browser_like([["PAGE ONE"], ["PAGE TWO"], ["PAGE THREE"]]))
|
||||
self.assertEqual(out.split("\f"), ["PAGE ONE", "PAGE TWO", "PAGE THREE"])
|
||||
|
||||
def test_form_xobject_text_is_read(self):
|
||||
out = pt.extract_text(browser_like([["BODY"]], xobject=True))
|
||||
self.assertIn("BODY", out)
|
||||
self.assertIn("FOOTER", out)
|
||||
|
||||
def test_object_streams(self):
|
||||
# The page tree and font live inside an /ObjStm (PDF 1.5+ writers).
|
||||
cmap_raw = zlib.compress(CMAP)
|
||||
content = zlib.compress(b"BT /F1 12 Tf " + _enc("PACKED") + b" Tj ET")
|
||||
inner = [
|
||||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
b"<< /Type /Pages /Count 1 /Kids [3 0 R] >>",
|
||||
b"<< /Type /Page /Parent 2 0 R /Contents 6 0 R "
|
||||
b"/Resources << /Font << /F1 4 0 R >> >> >>",
|
||||
b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /ToUnicode 5 0 R >>",
|
||||
]
|
||||
header, body, off = b"", b"", 0
|
||||
for i, o in enumerate(inner, start=1):
|
||||
header += b"%d %d " % (i, off)
|
||||
body += o + b" "
|
||||
off += len(o) + 1
|
||||
packed = zlib.compress(header + body)
|
||||
doc = (b"%PDF-1.7\n"
|
||||
+ (b"5 0 obj\n<< /Filter /FlateDecode /Length %d >>\nstream\n" % len(cmap_raw))
|
||||
+ cmap_raw + b"\nendstream\nendobj\n"
|
||||
+ (b"6 0 obj\n<< /Filter /FlateDecode /Length %d >>\nstream\n" % len(content))
|
||||
+ content + b"\nendstream\nendobj\n"
|
||||
+ (b"7 0 obj\n<< /Type /ObjStm /N 4 /First %d /Filter /FlateDecode /Length %d >>\nstream\n"
|
||||
% (len(header), len(packed)))
|
||||
+ packed + b"\nendstream\nendobj\n%%EOF\n")
|
||||
self.assertEqual(pt.extract_text(doc), "PACKED")
|
||||
|
||||
def test_extract_pdf_builtin_uses_it(self):
|
||||
self.assertEqual(ex.extract_pdf_builtin(browser_like([["HELLO"]])), "HELLO")
|
||||
|
||||
def test_hostile_input_never_raises(self):
|
||||
for junk in (b"", b"%PDF-1.4\n", b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 1 0 R >> endobj",
|
||||
b"%PDF-1.4\n1 0 obj << /Length 99999999 >>\nstream\n\x78\x9c\xff\xff\nendstream\nendobj"):
|
||||
ex.extract_pdf_builtin(junk) # returns None or text; must not raise
|
||||
|
||||
def test_reference_cycles_do_not_hang(self):
|
||||
doc = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n"
|
||||
b"2 0 obj << /Type /Pages /Kids [2 0 R] >> endobj\n%%EOF")
|
||||
self.assertIsNone(pt.extract_text(doc))
|
||||
|
||||
|
||||
HELVETICA = (b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /FirstChar 32 /LastChar 126 "
|
||||
b"/Widths [" + b" ".join([b"500"] * 95) + b"] >>")
|
||||
|
||||
|
||||
def simple_doc(streams, font=HELVETICA, objects=(), page_extra=b""):
|
||||
"""One page, font /F1, content from `streams` (each its own stream in
|
||||
/Contents), `objects` appended as-is after the page."""
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") # 1
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>") # 2
|
||||
pdf.add(font) # 3
|
||||
first = len(pdf.objs) + 2
|
||||
refs = b" ".join(b"%d 0 R" % (first + i) for i in range(len(streams)))
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> %s >> "
|
||||
b"/Contents [%s] >>" % (page_extra, refs)) # 4
|
||||
for s in streams:
|
||||
pdf.add(b"<< >>", s)
|
||||
for o in objects:
|
||||
pdf.objs.append(o)
|
||||
return pdf.bytes()
|
||||
|
||||
|
||||
class HostileInput(unittest.TestCase):
|
||||
"""Review B1/B2: a user's file is arbitrary bytes. Every shape here once
|
||||
raised past the extractor or ran without bound."""
|
||||
|
||||
def test_kids_that_is_not_an_array(self):
|
||||
doc = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n"
|
||||
b"2 0 obj << /Type /Pages /Kids 5 >> endobj\n%%EOF")
|
||||
self.assertIsNone(pt.extract_text(doc))
|
||||
|
||||
def test_objstm_offsets_that_are_not_integers(self):
|
||||
packed = zlib.compress(b"1 1.e999 2 1e5 << /Type /Catalog >> ")
|
||||
doc = (b"%PDF-1.7\n" + b"7 0 obj\n<< /Type /ObjStm /N 2 /First 14 /Filter /FlateDecode "
|
||||
b"/Length %d >>\nstream\n" % len(packed) + packed + b"\nendstream\nendobj\n%%EOF")
|
||||
self.assertIsNone(pt.extract_text(doc))
|
||||
|
||||
def test_bfrange_destination_past_its_width(self):
|
||||
cmap = (b"1 begincodespacerange <0000> <FFFF> endcodespacerange\n"
|
||||
b"1 beginbfrange <0000> <FFFF> <FFF0> endbfrange\n")
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>")
|
||||
tu = pdf.add(b"<< >>", cmap)
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu)
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 6 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf <00000001> Tj ET")
|
||||
# In-range codes map (U+FFF0, U+FFF1); the run past 0xFFFF stops, never raises.
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()), "\ufff0\ufff1")
|
||||
|
||||
def test_deep_nesting_is_skipped_not_a_recursion_error(self):
|
||||
doc = (b"%PDF-1.4\n9 0 obj " + b"[" * 100000 + b" endobj\n"
|
||||
+ simple_doc([b"BT /F1 12 Tf (STILL READ) Tj ET"])[len(b"%PDF-1.7\n"):])
|
||||
self.assertEqual(pt.extract_text(doc), "STILL READ")
|
||||
|
||||
def test_form_fan_out_is_bounded(self):
|
||||
# X1 draws X2 ten times, X2 draws X3 ten times … six levels: 10^6
|
||||
# invocations unbounded. The budget stops it quickly with the text
|
||||
# read so far.
|
||||
objs, first = [], 6
|
||||
for lvl in range(6):
|
||||
num = first + lvl
|
||||
if lvl < 5:
|
||||
body = b"BT /F1 12 Tf (L%d) Tj ET " % lvl + b"/X Do " * 10
|
||||
res = b"/Resources << /Font << /F1 3 0 R >> /XObject << /X %d 0 R >> >>" % (num + 1)
|
||||
else:
|
||||
body, res = b"BT /F1 12 Tf (LEAF) Tj ET", b"/Resources << /Font << /F1 3 0 R >> >>"
|
||||
objs.append(b"%d 0 obj\n<< /Type /XObject /Subtype /Form %s /Length %d >>\nstream\n"
|
||||
% (num, res, len(body)) + body + b"\nendstream\nendobj\n")
|
||||
doc = simple_doc([b"/X Do"], objects=objs, page_extra=b"/XObject << /X 6 0 R >>")
|
||||
t0 = time.monotonic()
|
||||
out = pt.extract_text(doc)
|
||||
self.assertLess(time.monotonic() - t0, 10)
|
||||
self.assertTrue(out.startswith("L0"))
|
||||
|
||||
def test_cmap_entries_are_bounded(self):
|
||||
cmap = b"1 begincodespacerange <0000> <FFFF> endcodespacerange\n" + \
|
||||
b"1 beginbfrange <0000> <FFFF> <0041> endbfrange\n" * 2000
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>")
|
||||
tu = pdf.add(b"<< >>", cmap)
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu)
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 6 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf <0000> Tj ET")
|
||||
b = pt.Budget()
|
||||
t0 = time.monotonic()
|
||||
pt.extract_text(pdf.bytes(), b)
|
||||
self.assertLess(time.monotonic() - t0, 10)
|
||||
self.assertEqual(b.reason, "cmap entries")
|
||||
|
||||
def test_a_stream_is_decoded_once(self):
|
||||
doc = simple_doc([b"BT /F1 12 Tf (ONCE) Tj ET /X Do /X Do /X Do /X Do"],
|
||||
objects=[b"6 0 obj\n<< /Type /XObject /Subtype /Form /Length 30 >>\nstream\n"
|
||||
b"BT /F1 12 Tf (FORM) Tj ET \nendstream\nendobj\n"],
|
||||
page_extra=b"/XObject << /X 6 0 R >>")
|
||||
with mock.patch.object(pt._Doc, "_decode", autospec=True, side_effect=pt._Doc._decode) as dec:
|
||||
self.assertIn("FORM", pt.extract_text(doc))
|
||||
self.assertEqual(sum(1 for c in dec.call_args_list if c.args[2].startswith(b"BT /F1 12 Tf (FORM)")), 1)
|
||||
|
||||
def test_missing_endobj_is_linear(self):
|
||||
doc = b"%PDF-1.4\n" + b"1 0 obj << /A 1 >>\n" * 60000 + b"endobj\n"
|
||||
t0 = time.monotonic()
|
||||
pt.extract_text(doc)
|
||||
self.assertLess(time.monotonic() - t0, 5)
|
||||
|
||||
def test_wall_time_budget(self):
|
||||
b = pt.Budget(seconds=0)
|
||||
out = pt.extract_text(browser_like([["A"]] * 30), b)
|
||||
self.assertEqual(b.reason, "time")
|
||||
self.assertIsNone(out)
|
||||
|
||||
def test_mutated_real_shapes_never_raise(self):
|
||||
rnd = random.Random(1)
|
||||
seeds = [browser_like([["HELLO WORLD"], ["PAGE TWO"]], xobject=True),
|
||||
simple_doc([b"BT /F1 12 Tf (plain) Tj ET"])]
|
||||
for _ in range(400):
|
||||
d = bytearray(rnd.choice(seeds))
|
||||
for _ in range(rnd.randint(1, 12)):
|
||||
i = rnd.randrange(len(d))
|
||||
d[i:i + rnd.randint(1, 4)] = bytes(rnd.randrange(256) for _ in range(rnd.randint(0, 4)))
|
||||
ex.extract_pdf_builtin(bytes(d)) # None or text; never raises
|
||||
|
||||
|
||||
class Layout(unittest.TestCase):
|
||||
"""Review non-blocking items: where words and lines break."""
|
||||
|
||||
def test_bt_resets_the_text_matrix(self):
|
||||
# Two blocks on the SAME line: Td in the second block is from the
|
||||
# origin again, not from where the first block left off.
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 50 700 Td (LEFT) Tj ET "
|
||||
b"BT /F1 10 Tf 200 700 Td (RIGHT) Tj ET"]))
|
||||
self.assertEqual(out, "LEFT RIGHT")
|
||||
|
||||
def test_words_placed_by_position_get_a_space_from_widths(self):
|
||||
# No space glyph: the gap between HELLO (5 x 500/1000 x 10 = 25 wide)
|
||||
# and a run moved 40 to the right is the only word boundary.
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HELLO) Tj 40 0 Td (WORLD) Tj ET"]))
|
||||
self.assertEqual(out, "HELLO WORLD")
|
||||
|
||||
def test_runs_that_continue_a_word_stay_joined(self):
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HEL) Tj 15 0 Td (LO) Tj ET"]))
|
||||
self.assertEqual(out, "HELLO")
|
||||
|
||||
def test_without_widths_no_space_is_invented(self):
|
||||
font = b"<< /Type /Font /Subtype /Type1 /BaseFont /SomeFace >>"
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HEL) Tj 15 0 Td (LO) Tj ET"], font=font))
|
||||
self.assertEqual(out, "HELLO")
|
||||
|
||||
def test_cid_widths_from_w_array(self):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [6 0 R] >>")
|
||||
tu = pdf.add(b"<< >>", CMAP)
|
||||
desc = pdf.add(b"<< /Type /Font /Subtype /CIDFontType2 /DW 1000 /W [36 [600 600 600 600 600 600]] >>")
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /DescendantFonts [%d 0 R] "
|
||||
b"/ToUnicode %d 0 R >>" % (desc, tu))
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 5 0 R >> >> /Contents 7 0 R >>")
|
||||
# "AB" at 600/1000 x 10 = 12 wide; the next run 20 on is a new word.
|
||||
pdf.add(b"<< >>", b"BT /F1 10 Tf 0 0 Td " + _enc("AB") + b" Tj 20 0 Td " + _enc("CD") + b" Tj ET")
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()), "AB CD")
|
||||
|
||||
def test_winansi_encoding(self):
|
||||
font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf (caf\\351 \\223q\\224 \\200) Tj ET"], font=font))
|
||||
self.assertEqual(out, "caf\u00e9 \u201cq\u201d \u20ac")
|
||||
|
||||
def test_macroman_encoding(self):
|
||||
font = (b"<< /Type /Font /Subtype /Type1 /BaseFont /Times "
|
||||
b"/Encoding << /BaseEncoding /MacRomanEncoding >> >>")
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf (caf\\216) Tj ET"], font=font))
|
||||
self.assertEqual(out, "caf\u00e9")
|
||||
|
||||
def test_font_carries_across_a_pages_content_streams(self):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>")
|
||||
tu = pdf.add(b"<< >>", CMAP)
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /ToUnicode %d 0 R >>" % tu)
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> "
|
||||
b"/Contents [6 0 R 7 0 R] >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf")
|
||||
pdf.add(b"<< >>", _enc("SPLIT") + b" Tj ET")
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()), "SPLIT")
|
||||
|
||||
def test_incremental_update_later_definition_wins(self):
|
||||
doc = simple_doc([b"BT /F1 12 Tf (OLD) Tj ET"])
|
||||
body = b"BT /F1 12 Tf (NEW) Tj ET"
|
||||
doc += (b"5 0 obj\n<< /Length %d >>\nstream\n" % len(body)) + body + b"\nendstream\nendobj\n%%EOF\n"
|
||||
self.assertEqual(pt.extract_text(doc), "NEW")
|
||||
|
||||
def test_stream_holding_endobj_text_is_read_whole(self):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>")
|
||||
pdf.add(HELVETICA)
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf (a endobj b) Tj 0 -20 Td (AFTER) Tj ET", compress=False)
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()), "a endobj b\nAFTER")
|
||||
|
||||
|
||||
class ReviewRound2(unittest.TestCase):
|
||||
"""Second review: lexing is what repeats, so lexing is what is charged;
|
||||
one bad value costs one stream, not the document."""
|
||||
|
||||
def _pages_sharing(self, body, npages):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
kids = b" ".join(b"%d 0 R" % (5 + i) for i in range(npages))
|
||||
pdf.add(b"<< /Type /Pages /Count %d /Kids [%s] >>" % (npages, kids))
|
||||
pdf.add(HELVETICA)
|
||||
pdf.add(b"<< >>", body) # 4
|
||||
for _ in range(npages):
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> "
|
||||
b"/Contents 4 0 R >>")
|
||||
return pdf.bytes()
|
||||
|
||||
def test_one_stream_lexed_by_many_pages_is_bounded(self):
|
||||
doc = self._pages_sharing(b"BT /F1 12 Tf (hello) Tj ET " + b"1 " * 1_000_000, 40)
|
||||
b = pt.Budget(seconds=1)
|
||||
t0 = time.monotonic()
|
||||
out = pt.extract_text(doc, b)
|
||||
self.assertLess(time.monotonic() - t0, 5)
|
||||
self.assertEqual(b.reason, "time")
|
||||
self.assertTrue(out.startswith("hello"))
|
||||
|
||||
def test_token_cap(self):
|
||||
doc = self._pages_sharing(b"BT /F1 12 Tf (hi) Tj ET " + b"1 " * 50_000, 10)
|
||||
with mock.patch.object(pt, "MAX_TOKENS", 120_000):
|
||||
b = pt.Budget()
|
||||
pt.extract_text(doc, b)
|
||||
self.assertEqual(b.reason, "tokens")
|
||||
|
||||
def test_a_shared_tounicode_is_parsed_once(self):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [6 0 R] >>")
|
||||
tu = pdf.add(b"<< >>", CMAP) # 3
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) # 4
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) # 5
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> "
|
||||
b"/Contents 7 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf " + _enc("AB") + b" Tj /F2 12 Tf " + _enc("CD") + b" Tj ET")
|
||||
with mock.patch.object(pt._Font, "_parse", autospec=True, side_effect=pt._Font._parse) as parse:
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()), "ABCD")
|
||||
self.assertEqual(parse.call_count, 1)
|
||||
|
||||
def test_words_far_apart_without_widths_are_separated(self):
|
||||
font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"
|
||||
for ops in (b"BT /F1 10 Tf 50 700 Td (Name) Tj 150 0 Td (Value) Tj ET",
|
||||
b"BT /F1 10 Tf 1 0 0 1 50 700 Tm (Name) Tj 1 0 0 1 200 700 Tm (Value) Tj ET"):
|
||||
self.assertEqual(pt.extract_text(simple_doc([ops], font=font)), "Name Value")
|
||||
|
||||
def test_standard_font_without_widths_uses_its_real_widths(self):
|
||||
# Helvetica W is 944/1000: (WWWW) ends at 37.76 at size 10. The rest of
|
||||
# the word placed exactly there is the same word; a guessed 500 would
|
||||
# have read a 17.76-unit gap and split it.
|
||||
font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (WWWW) Tj 37.76 0 Td (ORD) Tj ET"],
|
||||
font=font))
|
||||
self.assertEqual(out, "WWWWORD")
|
||||
self.assertEqual(pt._standard_widths(b"ABCDEF+Arial,Bold")[1][ord("W") - 32], 944)
|
||||
self.assertEqual(pt._standard_widths(b"Courier-Oblique"), (32, [600] * 95))
|
||||
self.assertIsNone(pt._standard_widths(b"SomeFace"))
|
||||
|
||||
def test_unknown_font_without_widths_guesses_wide(self):
|
||||
font = b"<< /Type /Font /Subtype /Type1 /BaseFont /SomeFace >>"
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (WWWW) Tj 37.76 0 Td (ORD) Tj ET"],
|
||||
font=font))
|
||||
self.assertEqual(out, "WWWWORD")
|
||||
|
||||
def test_superscript_rise_is_not_a_new_line(self):
|
||||
out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (E=mc) Tj 20 3 Td (2) Tj "
|
||||
b"5 -3 Td ( is) Tj ET"]))
|
||||
self.assertEqual(out, "E=mc2 is")
|
||||
|
||||
def test_the_word_stream_inside_a_string_owns_no_bytes(self):
|
||||
# The stray object sits BEFORE every page object.
|
||||
doc = simple_doc([b"BT /F1 12 Tf (BODY) Tj ET"])
|
||||
head, rest = doc.split(b"1 0 obj", 1)
|
||||
doc = head + b"9 0 obj\n<< /Title (a live stream\n) >>\nendobj\n1 0 obj" + rest
|
||||
self.assertEqual(pt.extract_text(doc), "BODY")
|
||||
|
||||
def test_indirect_length(self):
|
||||
body = b"BT /F1 12 Tf (INDIRECT) Tj ET"
|
||||
raw = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n"
|
||||
b"2 0 obj << /Type /Pages /Count 1 /Kids [3 0 R] >> endobj\n"
|
||||
b"3 0 obj << /Type /Page /Parent 2 0 R /Contents 4 0 R >> endobj\n"
|
||||
b"4 0 obj << /Length 5 0 R >>\nstream\n" + body + b"\nendstream\nendobj\n"
|
||||
b"5 0 obj %d endobj\n%%%%EOF" % len(body))
|
||||
self.assertEqual(pt.extract_text(raw), "INDIRECT")
|
||||
|
||||
def test_one_bad_value_costs_one_stream_not_the_document(self):
|
||||
bad_font = b"<< /Type /Font /Subtype /Type1 /Encoding [/a /b] >>"
|
||||
self.assertEqual(pt.extract_text(simple_doc([b"BT /F1 12 Tf (still) Tj ET"], font=bad_font)),
|
||||
"still")
|
||||
huge = b"BT /F1 12 Tf (hello world) Tj [(a) 1" + b"0" * 400 + b" (b)] TJ ET"
|
||||
self.assertTrue(pt.extract_text(simple_doc([huge])).startswith("hello world"))
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 2 /Kids [4 0 R 5 0 R] >>")
|
||||
pdf.add(HELVETICA)
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 6 0 R >>")
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 7 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf (PAGE ONE) Tj ET")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf (PAGE TWO) Tj ET")
|
||||
real = pt._Reader.content
|
||||
calls = []
|
||||
|
||||
def first_page_breaks(self, data, *a, **k):
|
||||
calls.append(1)
|
||||
if len(calls) == 1:
|
||||
raise TypeError("a malformed value")
|
||||
return real(self, data, *a, **k)
|
||||
with mock.patch.object(pt._Reader, "content", first_page_breaks):
|
||||
self.assertEqual(pt.extract_text(pdf.bytes()).split("\f"), ["", "PAGE TWO"])
|
||||
|
||||
def test_cid_pages_without_a_map_do_not_fall_back_to_junk(self):
|
||||
pdf = Pdf()
|
||||
pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>")
|
||||
pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>")
|
||||
pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H >>")
|
||||
pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>")
|
||||
pdf.add(b"<< >>", b"BT /F1 12 Tf <00410042> Tj ET")
|
||||
self.assertIsNone(pt.extract_text(pdf.bytes()))
|
||||
|
||||
def test_index_checks_the_clock(self):
|
||||
data = b"%PDF-1.4\n" + b"".join(b"%d 0 obj [1 2 3] endobj\n" % i for i in range(200_000))
|
||||
b = pt.Budget(seconds=0)
|
||||
pt.extract_text(data, b)
|
||||
self.assertEqual(b.reason, "time")
|
||||
|
||||
|
||||
|
||||
class ReviewRound4(unittest.TestCase):
|
||||
"""Memory, not just time: a lexer step must not allocate per byte."""
|
||||
|
||||
def _peak(self, fn):
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
fn()
|
||||
peak = tracemalloc.get_traced_memory()[1]
|
||||
tracemalloc.stop()
|
||||
return peak
|
||||
|
||||
def test_spaced_hex_string_is_stripped_without_per_byte_objects(self):
|
||||
data = b"<" + b"0 " * 2_000_000 + b"41>"
|
||||
peak = self._peak(lambda: pt._Lexer(data).token())
|
||||
self.assertEqual(pt._Lexer(b"<4 1 4 2>").token(), ("str", b"AB"))
|
||||
self.assertLess(peak, 3 * len(data), peak)
|
||||
|
||||
def test_comment_lines_are_skipped_without_per_line_state(self):
|
||||
data = b"%\n" * 1_000_000 + b"(x)"
|
||||
peak = self._peak(lambda: pt._Lexer(data).token())
|
||||
self.assertEqual(pt._Lexer(data).token(), ("str", b"x"))
|
||||
self.assertLess(peak, len(data), peak)
|
||||
|
||||
def test_a_long_escape_run_checks_the_clock(self):
|
||||
b = pt.Budget(seconds=0)
|
||||
with self.assertRaises(pt._Spent):
|
||||
pt._Lexer(b"(" + b"\\a" * 300_000 + b")", budget=b).token()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user