#!/usr/bin/env python3
"""Reproducible synthetic PDF extraction sample. No network or paid APIs."""
from pathlib import Path
import csv, json, subprocess, re, hashlib, platform, sys
from decimal import Decimal
from importlib.metadata import version
from reportlab.pdfgen import canvas
from reportlab.lib import colors
from reportlab.lib.utils import ImageReader
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from PIL import Image, ImageOps
import pdfplumber

ROOT = Path(__file__).resolve().parent
FONT_DIR = Path('/usr/share/fonts/truetype/dejavu')
pdfmetrics.registerFont(TTFont('SampleSans',str(FONT_DIR/'DejaVuSans.ttf')))
pdfmetrics.registerFont(TTFont('SampleSansBold',str(FONT_DIR/'DejaVuSans-Bold.ttf')))
for d in ['fixtures', 'expected', 'observed', 'results', 'preview', 'guides']:
    (ROOT / d).mkdir(exist_ok=True)
HEAD = ['RecordID','Date','Description','Qty','UnitUSD','TotalUSD']
PRODUCTS = ['Cable clips','Desk pad','Paper tray','Pen holder','File folder','Label roll',
            'Binder set','Notebook pack','Storage box','Mouse mat','Card sleeve','Index tabs']
QUANTITIES = [2,3,1,4,5,2,7,3,2,6,4,8]
PRICES = ['12.50','8.99','105.00','0.75','24.80','19.95','3.20','17.45','42.10','9.99','2.35','6.40']
ROWS = [[f'R{i+1:03d}',f'2026-09-{i+1:02d}',PRODUCTS[i],str(QUANTITIES[i]),PRICES[i],
         f'{Decimal(PRICES[i])*QUANTITIES[i]:.2f}'] for i in range(12)]
PAGE_W, PAGE_H = 612, 792
LEFT, TOP, RH = 42, 624, 36
WIDTHS = [72,90,156,42,66,60]
BOUNDARIES = [LEFT]
for width in WIDTHS: BOUNDARIES.append(BOUNDARIES[-1]+width)

def write_csv(path, rows, header=True):
    with path.open('w',newline='',encoding='utf-8') as f:
        w = csv.writer(f,lineterminator='\n')
        if header: w.writerow(HEAD)
        w.writerows(rows)

def page(c, title, rows, number, count):
    c.setFillColor(colors.HexColor('#102C43')); c.rect(0,710,PAGE_W,82,fill=1,stroke=0)
    c.setFillColor(colors.white); c.setFont('SampleSansBold',19)
    c.drawString(42,750,'Soulkey PDF extraction sample')
    c.setFont('SampleSans',10); c.drawString(42,729,title)
    c.setFillColor(colors.HexColor('#1B3345')); c.setFont('SampleSansBold',12)
    c.drawString(42,678,'Synthetic purchasing records')
    c.setFont('SampleSans',9); c.drawString(42,658,'All records are invented. Amounts are USD. This is a test fixture, not an invoice.')
    for ri, row in enumerate([HEAD]+rows):
        y = TOP - (ri+1)*RH
        c.setFillColor(colors.HexColor('#E4EEF3') if ri==0 else colors.white)
        c.rect(LEFT,y,sum(WIDTHS),RH,fill=1,stroke=0)
        c.setFillColor(colors.HexColor('#172D3D')); c.setFont('SampleSansBold' if ri==0 else 'SampleSans',9)
        for ci,value in enumerate(row): c.drawString(BOUNDARIES[ci]+6,y+14,value)
    c.setStrokeColor(colors.HexColor('#788C9A')); c.setLineWidth(.65)
    bottom = TOP-(len(rows)+1)*RH
    for x in BOUNDARIES: c.line(x,bottom,x,TOP)
    for ri in range(len(rows)+2): c.line(LEFT,TOP-ri*RH,BOUNDARIES[-1],TOP-ri*RH)
    c.setFont('SampleSans',8); c.setFillColor(colors.HexColor('#526876'))
    c.drawString(42,84,'Six data columns. A repeated header is metadata and must not become a record.')
    c.drawString(42,67,'Fixture version 1.0 - generated 2026-09-30 - no real customer information')
    c.drawRightString(570,48,f'Page {number} of {count}')
    c.showPage()

def make_pdf(path, title, chunks):
    c = canvas.Canvas(str(path),pagesize=(PAGE_W,PAGE_H),invariant=1)
    c.setTitle(title); c.setAuthor('Soulkey Research')
    for i,chunk in enumerate(chunks): page(c,title,chunk,i+1,len(chunks))
    c.save()

make_pdf(ROOT/'fixtures/01_digital_table.pdf','Digital text table', [ROWS[:6]])
make_pdf(ROOT/'fixtures/02_repeated_headers.pdf','Two-page table with a repeated header', [ROWS[:6],ROWS[6:]])
# The scan has exactly the same visible page content as the digital fixture.
# Rasterize at 200 DPI, convert to grayscale, then embed only the image.
subprocess.run(['pdftoppm','-f','1','-singlefile','-r','200','-png',
                str(ROOT/'fixtures/01_digital_table.pdf'),str(ROOT/'preview/scan_source')],check=True)
im = ImageOps.grayscale(Image.open(ROOT/'preview/scan_source.png'))
im.save(ROOT/'preview/scan_source.png')
c = canvas.Canvas(str(ROOT/'fixtures/03_scanned_table.pdf'),pagesize=(PAGE_W,PAGE_H),invariant=1)
c.setTitle('Image-only scan of the digital table'); c.setAuthor('Soulkey Research')
c.drawImage(ImageReader(im),0,0,PAGE_W,PAGE_H); c.showPage(); c.save()

cases = {'01_digital_table':ROWS[:6], '02_repeated_headers':ROWS, '03_scanned_table':ROWS[:6]}
for case, rows in cases.items(): write_csv(ROOT/f'expected/{case}.csv',rows)

def normalize(row): return [' '.join(str(v or '').split()) for v in row]

def score(rows, expected):
    # Preserve source order and compare each position exactly. Report record-ID
    # integrity separately so a mistyped ID does not erase other correct cells.
    byid = {}
    for row in rows:
        if row: byid.setdefault(row[0],[]).append(row)
    matched = 0; complete = 0; errors = []; duplicates = 0
    for ri,exp in enumerate(expected):
        candidates = byid.get(exp[0],[])
        if len(candidates)>1: duplicates += len(candidates)-1
        got = rows[ri] if ri<len(rows) else []
        good = 0
        for i, want in enumerate(exp):
            actual = got[i] if i<len(got) else ''
            if actual==want: matched+=1; good+=1
            else: errors.append({'record':exp[0], 'field':HEAD[i], 'expected':want,'actual':actual})
        complete += good==len(HEAD)
    expected_ids = {r[0] for r in expected}
    extras = sum(1 for r in rows if not r or r[0] not in expected_ids)
    return {'expected_rows':len(expected),'output_rows':len(rows),'exact_rows':complete,
            'correct_cells':matched,'expected_cells':len(expected)*len(HEAD),
            'extra_rows':max(0,len(rows)-len(expected)),
            'unexpected_ids':extras,'matched_ids':sum(r[0] in byid for r in expected),
            'duplicate_ids':duplicates,'errors':errors}

results=[]; raw_headers={}
for case,expected in cases.items():
    path=ROOT/f'fixtures/{case}.pdf'
    extracted=[]
    with pdfplumber.open(path) as pdf:
        for pg in pdf.pages:
            for table in pg.extract_tables(): extracted.extend(normalize(r) for r in table)
        char_count = sum(len(pg.chars) for pg in pdf.pages)
    (ROOT/f'observed/{case}_pdfplumber_raw.json').write_text(json.dumps(extracted,indent=2)+'\n')
    headers = sum(r==HEAD for r in extracted)
    raw_headers[case]={'detected_text_characters':char_count,'extracted_header_rows':headers,
                     'rows_after_removing_first_header_only':max(len(extracted)-1,0)}
    if case=='02_repeated_headers':
        write_csv(ROOT/'observed/02_repeated_headers_naive_drop_first.csv',extracted[1:])
    cleaned=[r for r in extracted if r!=HEAD]
    write_csv(ROOT/f'observed/{case}_pdfplumber.csv',cleaned)
    results.append({'case':case,'method':'pdfplumber tables + exact-header filter',**score(cleaned,expected)})
    subprocess.run(['pdftotext','-layout',str(path),str(ROOT/f'observed/{case}_pdftotext.txt')],check=True)
    txt=(ROOT/f'observed/{case}_pdftotext.txt').read_text()
    # A disclosed template-aware parser: only rows beginning with Rddd and
    # exactly six whitespace-delimited columns are admitted. No values repaired.
    rows=[]; rejected=[]
    for line in txt.splitlines():
        if re.match(r'^\s*R\d{3}\b',line):
            cols=re.split(r'\s{2,}',line.strip())
            if len(cols)==6: rows.append(cols)
            else: rejected.append(line)
    write_csv(ROOT/f'observed/{case}_pdftotext.csv',rows)
    (ROOT/f'observed/{case}_pdftotext_rejected.json').write_text(json.dumps(rejected,indent=2)+'\n')
    results.append({'case':case,'method':'pdftotext -layout + six-column parser',**score(rows,expected)})

# OCR route is deliberately template-guided, not an automatic table detector.
# Each body cell is cropped using the known shared table geometry. It uses no
# expected values, character substitutions, whitelist or manual repair.
ocr_rows=[]; raw_ocr=[]
for ri in range(6):
    values=[]
    for ci in range(6):
        scale=200/72
        ytop=PAGE_H-(TOP-(ri+1)*RH)
        # remove the rule border while retaining all cell content
        box=(round((BOUNDARIES[ci]+3)*scale),round((ytop+3)*scale),
             round((BOUNDARIES[ci+1]-3)*scale),round((ytop+RH-3)*scale))
        crop=im.crop(box)
        crop_path=ROOT/f'preview/ocr_cell_{ri:02d}_{ci:02d}.png'; crop.save(crop_path)
        proc=subprocess.run(['tesseract',str(crop_path),'stdout','--psm','7','-l','eng'],
                            text=True,capture_output=True,check=True)
        raw=proc.stdout; values.append(' '.join(raw.split()))
        raw_ocr.append({'row':ri+1,'column':ci+1,'crop_pixels':box,'text':raw})
    ocr_rows.append(values)
write_csv(ROOT/'observed/03_scanned_table_tesseract_template.csv',ocr_rows)
(ROOT/'observed/03_scanned_table_tesseract_raw.json').write_text(json.dumps(raw_ocr,indent=2)+'\n')
results.append({'case':'03_scanned_table','method':'Tesseract 200-DPI template-guided cell OCR',**score(ocr_rows,ROWS[:6])})

versions={'python':platform.python_version(),'platform':platform.platform(),
          'pdfplumber':version('pdfplumber'),'reportlab':version('reportlab'),'Pillow':version('Pillow'),
          'tesseract':subprocess.run(['tesseract','--version'],capture_output=True,text=True).stdout.splitlines()[0],
          'pdftotext':subprocess.run(['pdftotext','-v'],capture_output=True,text=True).stderr.splitlines()[0]}
payload={'sample_version':'1.0','test_date':'2026-09-30','versions':versions,
         'raw_header_checks':raw_headers,'results':results,
         'untested':['PDF.co','Adobe Acrobat','other commercial tools'],
         'limitations':['Three clean synthetic fixtures, not a representative real-world benchmark.',
                        'OCR uses known fixed cell geometry and a clean 200-DPI image; no skew/noise/handwriting.',
                        'Template-aware post-processing is part of every scored route.',
                        'No speed, cost, security, cloud retention or vendor ranking conclusion is supported.']}
(ROOT/'results/results.json').write_text(json.dumps(payload,indent=2)+'\n')
fields=['case','method','expected_rows','output_rows','exact_rows','correct_cells','expected_cells','matched_ids','unexpected_ids','extra_rows','duplicate_ids']
with (ROOT/'results/results.csv').open('w',newline='') as f:
    w=csv.DictWriter(f,fields,extrasaction='ignore',lineterminator='\n'); w.writeheader();w.writerows(results)

# Full-page previews for visual QA. Originals and the scan are not altered.
for case in cases:
    subprocess.run(['pdftoppm','-r','100','-png',str(ROOT/f'fixtures/{case}.pdf'),str(ROOT/f'preview/{case}')],check=True)
manifest={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest()
          for folder in ['fixtures','expected','observed','results'] for p in sorted((ROOT/folder).iterdir())
          if p.is_file() and p.name not in ['sha256.json','run-log.json']}
(ROOT/'results/sha256.json').write_text(json.dumps(manifest,indent=2)+'\n')
print(json.dumps(payload,indent=2))
