#!/usr/bin/env python3
#
# LDASTools frameCPP - Generate C++ source with hard-coded PDF definitions
#
# Copyright (C) 2025 California Institute of Technology
#

import argparse
import os
import re
import sys
from pathlib import Path
from collections import defaultdict

try:
    import pdfplumber
except ImportError:
    print("Error: pdfplumber is required. Install with: pip install pdfplumber", file=sys.stderr)
    sys.exit(1)


# Map version numbers to PDF files (for validation and reference)
# Hybrid approach: Version is dynamically detected from PDF content,
# this map validates detected version matches expected version
VERSION_PDF_MAP = {
    3: "T970130-B.pdf",
    4: "T970130-D.pdf",
    5: "T970130-E.pdf",
    6: "T970130-F.pdf",
    7: "T970130-G.pdf",
    8: "T970130-v2.pdf",
    9: "T970130-v4.pdf",
}

# Map object names to search patterns
OBJECT_PATTERNS = {
    "FrameH": [r"Frame Header Structure", r"FrameH Structure"],
    "FrAdcData": [r"ADC Data Structure", r"FrAdcData Structure"],
    "FrDetector": [r"Detector Data Structure", r"FrDetector Structure"],
    "FrEndOfFile": [r"End Of File.*?Structure", r"FrEndOfFile Structure"],
    "FrEndOfFrame": [r"End Of Frame.*?Structure", r"FrEndOfFrame Structure"],
    "FrEvent": [r"Event Data Structure", r"FrEvent Structure"],
    "FrHistory": [r"History.*?Structure", r"FrHistory Structure"],
    "FrMsg": [r"Message.*?Structure", r"FrMsg Structure"],
    "FrProcData": [r"Processed Data Structure", r"FrProcData Structure"],
    "FrRawData": [r"Raw Data Structure", r"FrRawData Structure"],
    "FrSE": [r"Structure Element", r"FrSE Structure"],
    "FrSerData": [r"Serial Data Structure", r"FrSerData Structure"],
    "FrSH": [r"Structure Header", r"FrSH Structure"],
    "FrSimData": [r"Simulation Data Structure", r"FrSimData Structure"],
    "FrSimEvent": [r"Simulation Event Structure", r"FrSimEvent Structure"],
    "FrStatData": [r"Static Data Structure", r"FrStatData Structure"],
    "FrSummary": [r"Summary.*?Structure", r"FrSummary Structure"],
    "FrTable": [r"Table Data Structure", r"FrTable Structure"],
    "FrTOC": [r"Table of Contents Structure", r"FrTOC Structure"],
    "FrTrigData": [r"Trigger Data Structure", r"FrTrigData Structure"],
    "FrVect": [r"Vector Data Structure", r"FrVect Structure"],
}

# Hidden fields that appear in all version 3 structures (common header fields)
HIDDEN_FIELDS_V3 = {'length', 'class', 'instance'}

# Known typos in PDF specifications that need correction
# Format: (version, object_name, field_name): corrected_field_name
# For duplicate field names, use: (version, object_name, field_name, occurrence_number): corrected_field_name
PDF_TYPO_CORRECTIONS = {
    (3, 'FrDetector', 'arrmXazimuth'): 'armXazimuth',  # Version 3 has extra 'm'
    (4, 'FrAdcData', 'timeOffsetSc'): 'timeOffsetS',  # Version 4 has footnote marker 'c'
    (4, 'FrAdcData', 'timeOffsetNc'): 'timeOffsetN',  # Version 4 has footnote marker 'c'
    (8, 'FrTOC', 'beginning'): 'positionADC',  # Version 8 has wrong field name
    (8, 'FrTOC', 'nProc', 2): 'positionProc',  # Version 8 duplicate nProc, 2nd occurrence should be positionProc
    (8, 'FrTOC', 'nSim', 2): 'positionSim',  # Version 8 duplicate nSim, 2nd occurrence should be positionSim
    (8, 'FrTOC', 'nSer', 2): 'positionSer',  # Version 8 duplicate nSer, 2nd occurrence should be positionSer
    (8, 'FrTOC', 'nFrame', 2): 'positionSum',  # Version 8 duplicate nFrame, 2nd occurrence should be positionSum
    (9, 'FrVect', 'dataValidComp'): 'dataValidCompScheme',  # Version 9 truncated field name
    (9, 'FrVect', 'nDataValidComp'): 'nDataValidCompBytes',  # Version 9 truncated field name
    (9, 'FrVect', 'ValidComp'): 'dataValid',  # Version 9 parser extracted array dimension as field name
    (9, 'FrTOC', 'bytes'): 'positionADC',  # Version 9 parser extracted comment word as field name
    (9, 'FrTOC', 'beginning', 1): 'positionProc',  # Version 9 parser extracted comment word, 1st occurrence
    (9, 'FrTOC', 'beginning', 2): 'positionSim',  # Version 9 parser extracted comment word, 2nd occurrence
    (9, 'FrTOC', 'beginning', 3): 'positionSer',  # Version 9 parser extracted comment word, 3rd occurrence
    (9, 'FrTOC', 'beginning', 4): 'positionSum',  # Version 9 parser extracted comment word, 4th occurrence
}

# Known type errors in PDF specifications that need correction
# Format: (version, object_name, field_name): corrected_type
PDF_TYPE_CORRECTIONS = {
    (3, 'FrStatData', 'detector'): 'PTR_STRUCT(FrDetector *)',  # PDF incorrectly shows FrVect *
    (8, 'FrProcData', 'table'): 'PTR_STRUCT(FrTable *)',  # PTR_STRUCT appears on next page, parser misses it
    (8, 'FrProcData', 'chkSum'): 'INT_4U',  # Parser confused by cross-page PTR_STRUCT, shows wrong type
    (8, 'FrTOC', 'positionADC'): 'INT_8U[nADC][nFrame]',  # PDF shows wrong type
    (8, 'FrTOC', 'positionProc'): 'INT_8U[nProc][nFrame]',  # PDF shows wrong type
    (8, 'FrTOC', 'positionSim'): 'INT_8U[nSim][nFrame]',  # PDF shows wrong type
    (8, 'FrTOC', 'positionSer'): 'INT_8U[nSer][nFrame]',  # PDF shows wrong type
    (8, 'FrTOC', 'positionSum'): 'INT_8U[nSummary][nFrame]',  # PDF shows wrong type
    (9, 'FrVect', 'dataValid'): 'CHAR[nDataValidCompBytes]',  # Array dimension split across multiple lines in PDF
    (9, 'FrTOC', 'positionADC'): 'INT_8U[nADC][nFrame]',  # Second array dimension split across lines
    (9, 'FrTOC', 'positionProc'): 'INT_8U[nProc][nFrame]',  # Second array dimension split across lines
    (9, 'FrTOC', 'positionSim'): 'INT_8U[nSim][nFrame]',  # Second array dimension split across lines
    (9, 'FrTOC', 'positionSer'): 'INT_8U[nSer][nFrame]',  # Second array dimension split across lines
    (9, 'FrTOC', 'positionSum'): 'INT_8U[nSummary][nFrame]',  # Second array dimension split across lines
}

# Known comment errors in PDF specifications that need correction
# Format: (version, object_name, field_name): corrected_comment
PDF_COMMENT_CORRECTIONS = {
    (8, 'FrProcData', 'history'): 'Channel history .',  # Comment merged with next field due to cross-page PTR_STRUCT
    (8, 'FrTOC', 'positionADC'): 'Array of lists of FrAdcData offset positions, in bytes, from beginning of file (size of nADC*nFrame) The ordering of entries:row/column follows the C convention. All positions for one ADC appear sequentially.',  # Corrected description
    (8, 'FrTOC', 'positionProc'): 'Array of FrProcData positions, in bytes, from beginning of file (size of nProc*nFrame). Ordering of entries:row/column: all positions for one FrProcData appear sequentially.',  # Corrected description
    (8, 'FrTOC', 'positionSim'): 'Array of FrSimData positions, in bytes, from beginning of file (size of nSim*nFrame). Ordering of entries:row/column: all positions for one FrSimData appear sequentially.',  # Corrected description
    (8, 'FrTOC', 'positionSer'): 'Array of FrSerData positions, in bytes, from beginning of file (size of nSer*nFrame). Ordering of entries:row/column: all positions for one FrSerData appear sequentially.',  # Corrected description
    (8, 'FrTOC', 'positionSum'): 'Array of FrSummary positions, in bytes, from beginning of file (size of nSummary*nFrame). Ordering of entries:row/column: all positions for one FrSummary appear sequentially.',  # Corrected description
}

# Fields to remove (bogus fields created by parser errors)
# Format: (version, object_name, field_name)
PDF_REMOVE_FIELDS = {
    (8, 'FrProcData', 'should'),  # Bogus field from footnote text, created by cross-page PTR_STRUCT confusion
}

# Fields that were skipped by the parser (e.g., due to footnote markers)
# Format: (version, object_name, after_field): (field_name, field_type, description)
# NOTE: This dictionary should rarely be needed - the parser now automatically strips
#       footnote markers during field extraction. Only use this for edge cases where
#       the footnote marker completely breaks the field parsing.
PDF_MISSING_FIELDS = {
    # Fields that were completely missed by the parser
    # Format: (version, object_name, field_after_which_to_insert): (field_name, field_type, field_description)
    (8, 'FrProcData', 'history'): ('next', 'PTR_STRUCT(FrProcData *)', 'Identifier for next FrProcData structure in the linked list.'),
    (9, 'FrAdcData', 'channelNumber'): ('nBits', 'INT_4U', 'Number of bits in A/D output'),
}

# Structures that were removed in specific versions
# Format: (version, object_name)
# These structures appear in the PDF (e.g., in appendices) but should not be included
PDF_REMOVED_STRUCTURES = {
    (9, 'FrStatData'),  # Removed in version 9, documented in Appendix E: Legacy Data Structures
}


def normalize_type(type_str: str, wrap_pointers: bool = False) -> str:
    """
    Normalize type syntax to canonical frameCPP format.

    Handles differences between manual PDFs (v3-8) and LaTeX PDFs (v9+):
    - "INT 4U" -> "INT_4U"
    - "REAL 8" -> "REAL_8"
    - "(FrVect *)" -> "PTR_STRUCT(FrVect *)" (only if wrap_pointers=True)

    Args:
        type_str: Type string as extracted from PDF
        wrap_pointers: If True, wrap raw pointers in PTR_STRUCT.
                      Set to False during extraction (parser handles it),
                      True for final normalization.

    Returns:
        Normalized type string in canonical frameCPP format
    """
    import re

    if not type_str:
        return type_str

    # Rule 1: Convert space to underscore in primitive types
    # "INT 4U" -> "INT_4U", "REAL 8" -> "REAL_8", "CHAR 2" -> "CHAR_2"
    # This is safe to do always, for all versions
    type_str = re.sub(r'\b(INT|REAL|CHAR)\s+(\d+)([US]?)\b', r'\1_\2\3', type_str)

    # Rule 2: Wrap raw pointers in PTR_STRUCT
    # "(FrVect *)" -> "PTR_STRUCT(FrVect *)"
    # Only if not already wrapped AND wrap_pointers=True
    # NOTE: During extraction, we skip this because the parser already handles
    # PTR_STRUCT wrapping correctly in the PTR_STRUCT continuation line logic
    if wrap_pointers:
        if type_str.startswith('(') and type_str.endswith(')') and '*' in type_str:
            if not type_str.startswith('PTR_STRUCT('):
                inner = type_str[1:-1]  # Remove outer parens
                type_str = f'PTR_STRUCT({inner})'

    # Rule 3: Normalize array syntax spacing
    # "INT 4U [nFrame]" -> "INT_4U[nFrame]"
    type_str = re.sub(r'\s+\[', '[', type_str)

    # Rule 4: Handle nested PTR_STRUCT in arrays
    # "PTR_STRUCT(Type *) [n]" -> "PTR_STRUCT(Type *)[n]"
    type_str = re.sub(r'\)\s+\[', ')[', type_str)

    # Rule 5: Normalize pointer spacing within PTR_STRUCT
    # Ensure consistent space before asterisk: "Type*" -> "Type *"
    # PTR_STRUCT(FrTable*) -> PTR_STRUCT(FrTable *)
    # PTR_STRUCT(FrVect  *) -> PTR_STRUCT(FrVect *)
    if 'PTR_STRUCT(' in type_str:
        # Ensure exactly one space before asterisk in pointer types
        type_str = re.sub(r'(\w)\s*\*\)', r'\1 *)', type_str)

    return type_str


# Map long names (from List of Tables) to short structure names
LONG_NAME_TO_STRUCT = {
    'Frame Header': 'FrameH',
    'ADC Data': 'FrAdcData',
    'Detector Data': 'FrDetector',
    'End of File Data': 'FrEndOfFile',
    'End of File Structure': 'FrEndOfFile',  # Variant used in section headers
    'End of Frame Data': 'FrEndOfFrame',
    'Event Data': 'FrEvent',
    'History': 'FrHistory',
    'Frame History Data': 'FrHistory',  # Variant used in section headers
    'History Data': 'FrHistory',  # Variant used in v4 section headers
    'Message Log Data': 'FrMsg',
    'Post-Processed Data': 'FrProcData',
    'Raw Data': 'FrRawData',
    'Serial Data': 'FrSerData',
    'Simulated Data': 'FrSimData',
    'Simulated Event Data': 'FrSimEvent',
    'Static Data': 'FrStatData',
    'Summary Data': 'FrSummary',
    'Table Data': 'FrTable',
    'Table Structure': 'FrTable',  # Variant used in v4 section headers
    'Table of Contents Data': 'FrTOC',
    'Table of Contents Structure': 'FrTOC',  # Variant used in v4 section headers
    'Trigger Data': 'FrTrigData',
    'Vector Data': 'FrVect',
}


def identify_frame_version(pdf_path):
    """
    Identify the frame format version number from the PDF.

    Looks for patterns like:
    - 'This document specifies Frame Format Version Number 4'
    - 'Frame Format version number for this specification 9'
    """
    with pdfplumber.open(pdf_path) as pdf:
        # Check first 20 pages for version specification
        for page in pdf.pages[:20]:
            text = page.extract_text()
            if not text:
                continue

            # Pattern 1: "This document specifies Frame Format Version Number X"
            match = re.search(r'This document specifies Frame Format Version Number (\d+)', text, re.IGNORECASE)
            if match:
                version = int(match.group(1))
                return version

            # Pattern 2: "Frame Format version number for this specification X"
            match = re.search(r'Frame Format version number for this specification[:\s]+(\d+)', text, re.IGNORECASE)
            if match:
                version = int(match.group(1))
                return version

    return None


def extract_expected_structures(pdf_path):
    """
    Extract expected structure list from 'List of Tables' section.

    Returns list of structure names that should be present in this version.
    Used for validation.
    """
    structures = []
    in_list_of_tables = False

    with pdfplumber.open(pdf_path) as pdf:
        for page in pdf.pages[:10]:
            text = page.extract_text()
            if not text:
                continue

            if 'List of Tables' in text:
                in_list_of_tables = True

            if not in_list_of_tables:
                continue

            lines = text.split('\n')
            for line in lines:
                if 'Definition' not in line or 'Structure' not in line:
                    continue

                # Pattern 1: "Table X: Name Structure Definition" (older versions)
                match = re.search(r'Table\s+\d+[:\.]?\s+(.+?)\s+(?:Structure\s+)?Definition', line, re.IGNORECASE)
                if not match:
                    # Pattern 2: "X Name Structure Definition" (version 9)
                    match = re.search(r'^\s*\d+\s+(.+?)\s+(?:Structure\s+)?Definition', line, re.IGNORECASE)

                if match:
                    full_name = match.group(1).strip()

                    # Skip incomplete names
                    if full_name == 'Data' or len(full_name) < 3:
                        continue

                    # Map to structure name
                    struct_name = LONG_NAME_TO_STRUCT.get(full_name)
                    if struct_name:
                        if struct_name not in structures:
                            structures.append(struct_name)
                    else:
                        # Try partial match
                        for key, value in LONG_NAME_TO_STRUCT.items():
                            if key.lower() in full_name.lower():
                                if value not in structures:
                                    structures.append(value)
                                break

    return structures


# ===========================================================================
# Footnote Marker Pattern Fragments
# ===========================================================================
# Footnote markers appear in PDFs as superscripts but are extracted as regular text.
# They are typically:
#   - Numeric: 1-3 digits (25, 27, 123)
#   - Alphabetic: Single lowercase letter (a, b, c)

FOOTNOTE_NUMERIC = r'\d{1,3}'                                # 1-3 digits
FOOTNOTE_ALPHA = r'[a-z]'                                    # Single lowercase letter
FOOTNOTE_MARKER = f'(?:{FOOTNOTE_NUMERIC}|{FOOTNOTE_ALPHA})'  # Either numeric or alpha
OPT_FOOTNOTE = f'{FOOTNOTE_MARKER}?'                         # Optional footnote suffix

# Type pattern components for building readable regex patterns
TYPE_INT = r'INT[_ ]\d+[US]'          # INT_4U or INT 4U (space or underscore)
TYPE_REAL = r'REAL[_ ]\d+'            # REAL_8 or REAL 8
TYPE_STRING = r'STRING'
TYPE_CHAR = r'CHAR'
ARRAY_SUFFIX = r'(?:\[[^\]]+\])?'     # Optional array dimension like [nBytes], [nDim]


def strip_footnote_markers(text):
    """
    Remove trailing footnote markers from text.

    Footnote markers are digits or lowercase letters that appear at the end of
    type names or field names in PDFs.

    Examples:
        "CHAR[nBytes]27" -> "CHAR[nBytes]"
        "timeOffsetSc" -> "timeOffsetS"
        "data25" -> "data"
    """
    if not text:
        return text
    # Remove trailing footnote marker (numeric or alphabetic)
    text = re.sub(rf'{FOOTNOTE_MARKER}$', '', text)
    return text


def extract_table_from_page(page, object_name, is_continuation=False, version=None):
    """
    Extract table fields using line-by-line text parsing.

    Strategy:
    1. Find section header "Long Name -- StructureName"
    2. Find table header "Data class Variable Name" AFTER section header
    3. Parse lines after table header as field definitions
    4. Each field starts with a TYPE (INT_xU, REAL_x, STRING, etc.)
    5. Continuation lines (wrapped text) don't start with types

    Args:
        page: pdfplumber page object
        object_name: Name of structure to extract (e.g., "FrProcData")
        is_continuation: If True, skip header search and start from beginning
        version: PDF version number (used for version-specific parsing)
    """
    text = page.extract_text()
    if not text:
        return None

    # Split into lines
    lines = text.split('\n')

    # Find section header line index (may not exist if on next page)
    section_header_idx = None
    for i, line in enumerate(lines):
        # Look for "Long Name -- StructureName" pattern in this line
        long_names = [ln for ln, sn in LONG_NAME_TO_STRUCT.items() if sn == object_name]
        for long_name in long_names:
            pattern = rf'{re.escape(long_name)}\s*[-–—]+\s*{re.escape(object_name)}'
            if re.search(pattern, line, re.IGNORECASE):
                section_header_idx = i
                break
        if section_header_idx is not None:
            break

    # Find table header line "Data class Variable Name"
    # Note: Some PDFs split the header across multiple lines (e.g., "Data class" on one line,
    # "Variable Name" on another). Handle both cases.
    # If is_continuation=True, skip header search and start from beginning

    if is_continuation:
        # Continuation page: start parsing from beginning (line 0)
        table_header_idx = -1  # Will add 1 to get 0
    else:
        # Normal page: search for table header
        start_search = (section_header_idx + 1) if section_header_idx is not None else 0

        table_header_idx = None
        for i in range(start_search, len(lines)):
            # Check for table header (either on same line or split across lines)
            has_data_class = 'Data class' in lines[i]
            has_variable_name = 'Variable Name' in lines[i] or ('Variable' in lines[i-1] if i > 0 else False)

            if has_data_class and (has_variable_name or section_header_idx is not None):
                # Make sure this is the right table by checking for structure name nearby
                # Look in previous 5 lines for structure name (short or long name)
                found_struct_name = False
                for j in range(max(0, i-5), i):
                    # Check for short name (e.g., "FrameH")
                    if object_name in lines[j]:
                        found_struct_name = True
                        break
                    # Check for long name (e.g., "Frame Header")
                    long_names = [ln for ln, sn in LONG_NAME_TO_STRUCT.items() if sn == object_name]
                    for long_name in long_names:
                        if long_name in lines[j]:
                            found_struct_name = True
                            break
                    if found_struct_name:
                        break
                if found_struct_name or section_header_idx is not None:
                    table_header_idx = i
                    break

        if table_header_idx is None:
            return None

    # Parse fields starting after table header
    fields = []
    current_field = None

    # Type patterns to identify start of new field
    # Note: Version 9 (LaTeX-generated) uses spaces: "INT 4U" instead of "INT_4U"
    # Group 1 captures the clean type (without footnote marker)
    # OPT_FOOTNOTE is outside group 1 to match but not capture the footnote
    type_patterns = [
        (rf'^\s*(\*{TYPE_INT}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'PTR_INT'),     # *INT_4U27 or *INT 4U[n]a
        (rf'^\s*(\*{TYPE_REAL}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'PTR_REAL'),   # *REAL_8 or *REAL 8[n]
        (rf'^\s*(\*{TYPE_STRING}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'PTR_STRING'),  # *STRING or *STRING[n]
        (rf'^\s*(\*{TYPE_CHAR}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'PTR_CHAR'),   # *CHAR or *CHAR[n]
        (rf'^\s*({TYPE_INT}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'INT'),           # INT_4U25 or INT 4U[nData]27
        (rf'^\s*({TYPE_REAL}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'REAL'),         # REAL_8 or REAL 8[nDim]
        (rf'^\s*({TYPE_STRING}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'STRING'),     # STRING or STRING[n]
        (rf'^\s*({TYPE_CHAR}{ARRAY_SUFFIX}){OPT_FOOTNOTE}', 'CHAR'),         # CHAR27 or CHAR[nBytes]27
        (rf'^\s*(\([A-Z][a-zA-Z0-9]*\s*\*\)){OPT_FOOTNOTE}', 'PTR'),         # (FrVect *) or (FrTable *)a
        (r'^\s*(See footnote [a-z])', 'FOOTNOTE'),                           # Special case: "See footnote a data ..."
    ]

    for i in range(table_header_idx + 1, len(lines)):
        line = lines[i]

        # Stop at next section header
        if re.search(r'^\d+\.\d+\.\d+\.\d+\s+\w', line):  # Numeric section like "4.3.2.20"
            break

        # Also stop if we hit another structure's section header (e.g., "Frame Detector -- FrDetector")
        if any(re.search(rf'{re.escape(ln)}\s*[-–—]+\s*{re.escape(sn)}', line, re.IGNORECASE)
               for ln, sn in LONG_NAME_TO_STRUCT.items() if sn != object_name):
            break

        # Skip empty lines
        if not line.strip():
            continue

        # Check if this line starts a new field (TYPE line)
        is_type_line = False
        matched_type = None
        matched_full_len = 0  # Length of full match including footnote markers

        for pattern, type_class in type_patterns:
            match = re.match(pattern, line)
            if match:
                is_type_line = True
                matched_type = match.group(1)  # Clean type without footnote
                matched_full_len = match.end()  # Position after full match (including footnote)
                break

        # Check for PTR_STRUCT continuation
        # PTR_STRUCT may be alone or followed by continuation text (e.g., "PTR_STRUCT (typically...)")
        # Also handle PDF typo: PTR_STRUXT
        if (line.strip().startswith('PTR_STRUCT') or line.strip().startswith('PTR_STRUXT')) and current_field:
            # Format as PTR_STRUCT(<pointer>)
            # Strip outer parentheses from pointer type: (FrVect *) -> FrVect *
            pointer_type = current_field['type'].strip('()')

            # Normalize pointer spacing: ensure space before asterisk
            # FrTable* -> FrTable *
            # FrTable * -> FrTable *
            pointer_type = re.sub(r'(\w)\*', r'\1 *', pointer_type)

            current_field['type'] = f"PTR_STRUCT({pointer_type})"

            # If there's text after PTR_STRUCT, add it to the comment
            remainder = line.strip()[len('PTR_STRUCT'):].strip()
            if remainder:
                current_field['comment'] += ' ' + remainder
            continue

        if is_type_line:
            # Check if current_field already has name but no type (name-first multi-line format)
            # This happens when previous line had: "fieldName Description text"
            # and current line is: "TYPE more text"
            if current_field and current_field['name'] and current_field['type'] is None:
                # Add type to existing field
                current_field['type'] = normalize_type(matched_type)
                # Append rest of line to comment
                if type_class == 'FOOTNOTE':
                    rest_match = re.search(r'See footnote [a-z]\s+(.*)', line.strip())
                    rest_of_line = rest_match.group(1) if rest_match else ''
                else:
                    rest_of_line = line[matched_full_len:].strip()
                if rest_of_line:
                    current_field['comment'] += ' ' + rest_of_line
            else:
                # Save previous field
                if current_field and current_field['name']:
                    fields.append(current_field)

                # Special handling for "See footnote X" pattern
                if type_class == 'FOOTNOTE':
                    # Example line: "See footnote a data nData elements of specified class"
                    # The actual type should be CHAR[nBytes]
                    matched_type = 'CHAR[nBytes]'
                    # Extract rest of line after "See footnote X"
                    rest_match = re.search(r'See footnote [a-z]\s+(.*)', line.strip())
                    rest_of_line = rest_match.group(1) if rest_match else ''
                else:
                    # Start new field - use full match length (including footnote) for correct offset
                    # Example: line="CHAR[nBytes]27 data...", matched_type="CHAR[nBytes]", matched_full_len=14
                    # This gives rest_of_line=" data..." instead of "27 data..."
                    rest_of_line = line[matched_full_len:].strip()

                # Extract variable name (first identifier - can start with upper or lowercase)
                # Variable names like: name, GTimeS, ULeapS, nData, data, etc.
                # Note: Footnote markers appear on TYPES, not on field names, so no stripping needed
                var_match = re.search(r'^([a-zA-Z][a-zA-Z0-9_]*)', rest_of_line)
                if var_match:
                    var_name = var_match.group(1)

                    # Skip hidden fields
                    if var_name in HIDDEN_FIELDS_V3:
                        current_field = None
                        continue

                    comment = rest_of_line[len(var_name):].strip()
                    current_field = {
                        'name': var_name,
                        'type': normalize_type(matched_type),
                        'comment': comment
                    }
                else:
                    # No variable name on this line - might be on next line (Version 9 format)
                    # Example: "(FrVect *)" on one line, "type Identifier..." on next line
                    # Keep the type and wait for next line
                    current_field = {
                        'name': None,  # Will be filled from next line
                        'type': normalize_type(matched_type),
                        'comment': ''
                    }
        else:
            # Continuation line
            if current_field:
                # If current_field has no name yet (Version 9 multi-line format), extract it
                if current_field['name'] is None:
                    var_match = re.search(r'^([a-zA-Z][a-zA-Z0-9_]*)', line.strip())
                    if var_match:
                        current_field['name'] = var_match.group(1)
                        # Skip if hidden field
                        if current_field['name'] in HIDDEN_FIELDS_V3:
                            current_field = None
                            continue
                        # Rest of line is comment
                        current_field['comment'] = line.strip()[len(current_field['name']):].strip()
                    # If no variable name found, skip this line
                    continue

                # Check for name-first multi-line field definition (v5-8 FrTable pattern)
                # Pattern: fieldName Description text
                # Next line: TYPE more text
                # Example:
                #   columnName Names of the columns. The frame API must then copy these names into each FrVect
                #   STRING[nColumn] structure included in this FrTable
                #
                # IMPORTANT: Only apply for FrTable in versions 5-8 where this pattern is known to exist
                # For other objects/versions, this would incorrectly match continuation text like "in CAPITALS"
                #
                # Additional constraints:
                # 1. Only check if current_field is complete (has name AND type)
                # 2. Only for FrTable object in versions 5-8
                # 3. Description after field name must start with capital letter
                if (current_field['type'] is not None and  # Only if field has type (is complete)
                    object_name == 'FrTable' and  # Only for FrTable
                    version is not None and version >= 5 and version <= 8):  # Only versions 5-8
                    name_first_match = re.match(r'^([a-z][a-zA-Z0-9_]*)\s+([A-Z].+)', line.strip())
                    if name_first_match:
                        # Lookahead to check if next non-empty line is a TYPE line
                        next_is_type = False
                        for j in range(i + 1, len(lines)):
                            if lines[j].strip():  # First non-empty line
                                # Check if it matches any type pattern
                                for pattern, _ in type_patterns:
                                    if re.match(pattern, lines[j]):
                                        next_is_type = True
                                        break
                                break

                        if next_is_type:
                            # This is a name-first field definition
                            # Save current field and start new one with name only
                            if current_field['name']:
                                fields.append(current_field)

                            field_name = name_first_match.group(1)
                            if field_name not in HIDDEN_FIELDS_V3:
                                current_field = {
                                    'name': field_name,
                                    'type': None,  # Will be filled from next line
                                    'comment': name_first_match.group(2)
                                }
                            else:
                                current_field = None
                            continue

                # Normal continuation - append to comment
                # Skip lines that are table notes, not field continuations
                table_notes = [
                    r'^First\s+',                    # "First three elements...", "First elements are as shown in table X"
                    r'^NOTE:',                       # "NOTE: ..."
                    r'^The structure',               # "The structure types..."
                    r'^One or more of the pointers', # "One or more of the pointers to the structures below..."
                ]
                is_table_note = any(re.match(pattern, line.strip()) for pattern in table_notes)

                if not is_table_note:
                    current_field['comment'] += ' ' + line.strip()

    # Don't forget last field
    if current_field and current_field['name']:
        fields.append(current_field)

    return fields if fields else None


def is_table_continuation_without_header(page_text):
    """
    Detect if a page continues a table from previous page WITHOUT a repeated header.

    This handles cases where tables span multiple pages but the continuation page
    doesn't repeat the section header or "Data class Variable Name" table header.

    Detection strategy:
    1. Check if page starts with field definitions (type patterns)
    2. Ensure there's no new section header (like "4.3.2.11 Title")
    3. Look within first 10-15 lines for field-like content

    Returns:
        bool: True if page appears to be table continuation without header
    """
    if not page_text:
        return False

    lines = page_text.split('\n')

    # Check first 15 lines for field patterns and section headers
    has_field_definitions = False
    has_new_section_header = False

    for i, line in enumerate(lines[:15]):
        # Check for new section header patterns that indicate a NEW structure, not continuation:
        # 1. Numbered section: "4.3.2.11 Post-processed Data -- FrProcData"
        # 2. Table header: "Table 8: ADC Data Structure Definition"
        if re.match(r'^\d+\.\d+\.\d+\.\d+\s+\w', line):
            has_new_section_header = True
            break
        if re.match(r'^Table\s+\d+:', line):
            has_new_section_header = True
            break

        # Check for type patterns that indicate field definitions
        # These patterns match the start of field definition lines
        field_patterns = [
            r'^\s*INT[_ ]\d+[US]',          # INT_4U, INT 4U, etc.
            r'^\s*REAL[_ ]\d+',             # REAL_8, REAL 8, etc.
            r'^\s*STRING',                  # STRING
            r'^\s*CHAR',                    # CHAR or CHAR[...]
            r'^\s*\*INT[_ ]\d+[US]',        # *INT_4U (pointer)
            r'^\s*\*REAL[_ ]\d+',           # *REAL_8 (pointer)
            r'^\s*\*STRING',                # *STRING (pointer)
            r'^\s*\*CHAR',                  # *CHAR (pointer)
            r'^\s*\([A-Z][a-zA-Z0-9]*\s*\*\)',  # (FrVect *), (FrTable *), etc.
            r'^\s*PTR_STRUCT',              # PTR_STRUCT keyword
        ]

        # Check if line matches any field pattern
        if any(re.search(pattern, line) for pattern in field_patterns):
            has_field_definitions = True
            # Don't break - keep checking for section headers

    # Page is a continuation if it has field definitions but no new section header
    return has_field_definitions and not has_new_section_header


def find_structure_in_pdf(pdf_path, object_name, version=None):
    """
    Find and extract structure definition from PDF.

    Uses section header pattern: 'Long Name -- StructureName' as anchor point.
    Table may span multiple pages with repeated headers.

    Args:
        pdf_path: Path to PDF file
        object_name: Name of structure to extract
        version: PDF version number (used for version-specific parsing)
    """
    # Find the long name(s) for this structure
    long_names = [long_name for long_name, struct in LONG_NAME_TO_STRUCT.items()
                  if struct == object_name]

    if not long_names:
        # Fallback to old pattern-based search if no mapping exists
        long_names = [object_name]

    with pdfplumber.open(pdf_path) as pdf:
        for page_idx, page in enumerate(pdf.pages):
            text = page.extract_text()
            if not text:
                continue

            # Look for section header: "Long Name -- StructureName"
            # Pattern allows for variations in separator: --, —, -, etc.
            found = False
            for long_name in long_names:
                pattern = rf'{re.escape(long_name)}\s*[-–—]+\s*{re.escape(object_name)}'
                if re.search(pattern, text, re.IGNORECASE):
                    found = True
                    break

            if not found:
                continue

            # Found the section header! Try extracting table from this page
            all_fields = []
            fields = extract_table_from_page(page, object_name, is_continuation=False, version=version)
            if fields:
                all_fields.extend(fields)

            # Check if structure is complete by looking for chkSum as last field
            # If last field is chkSum, the structure is complete and shouldn't look for continuation
            # This handles cases where multiple structures appear on same page:
            # Example: v9 page 27 has FrHistory (complete with chkSum), FrMsg (complete with chkSum),
            # and FrProcData (incomplete, continues to page 28).
            structure_is_complete = False
            if all_fields and all_fields[-1]['name'] == 'chkSum':
                structure_is_complete = True

            # Only check continuation pages if structure is incomplete
            if not structure_is_complete:
                # Check next page(s) for table continuation
                # Look up to 3 pages ahead for continuation
                # Support two types of continuation:
                #   1. With repeated header (structure name + "Data class Variable Name")
                #   2. Without repeated header (just field definitions continuing)
                for offset in range(1, 4):
                    if page_idx + offset >= len(pdf.pages):
                        break

                    next_page = pdf.pages[page_idx + offset]
                    next_text = next_page.extract_text()
                    if not next_text:
                        continue

                    # Strategy 1: Check for traditional continuation with repeated header
                    # Look for structure name (short or long) near top of page
                    lines = next_text.split('\n')
                    has_struct_name = False
                    has_table_header = False

                    for i, line in enumerate(lines[:10]):  # Check first 10 lines
                        # Check for short name (e.g., "FrameH")
                        if object_name in line:
                            has_struct_name = True
                        # Check for long name (e.g., "Frame Header")
                        for long_name in long_names:
                            if long_name in line:
                                has_struct_name = True
                                break
                        if 'Data class' in line and 'Variable Name' in line:
                            has_table_header = True

                    # Check both continuation strategies
                    is_continuation_with_header = has_struct_name and has_table_header
                    is_continuation_without_header = is_table_continuation_without_header(next_text)

                    # If either strategy indicates continuation, extract fields
                    if is_continuation_with_header:
                        # Traditional continuation with repeated header
                        continuation_fields = extract_table_from_page(next_page, object_name, is_continuation=False, version=version)
                        if continuation_fields:
                            all_fields.extend(continuation_fields)
                            # Check if structure is now complete after adding continuation fields
                            if continuation_fields[-1]['name'] == 'chkSum':
                                break
                    elif is_continuation_without_header:
                        # Continuation without header - parse from beginning
                        continuation_fields = extract_table_from_page(next_page, object_name, is_continuation=True, version=version)
                        if continuation_fields:
                            all_fields.extend(continuation_fields)
                            # Check if structure is now complete after adding continuation fields
                            if continuation_fields[-1]['name'] == 'chkSum':
                                break
                    else:
                        # No continuation found, stop looking
                        break

            if all_fields:
                return {
                    'object_name': object_name,
                    'header': {'name': object_name, 'comment': ''},
                    'fields': all_fields
                }

    return None


def escape_cpp_string(s):
    """Escape a string for use in C++ source code."""
    s = s.replace('\\', '\\\\')
    s = s.replace('"', '\\"')
    s = s.replace('\n', '\\n')
    s = s.replace('\r', '\\r')
    s = s.replace('\t', '\\t')
    return s


def scan_all_pdfs(pdf_dir):
    """
    Scan all PDFs and extract all structure definitions.

    Uses hybrid approach:
    1. Dynamically detect version from PDF content (primary)
    2. Validate against VERSION_PDF_MAP (secondary)
    3. Warn on mismatch but use detected version
    """
    all_definitions = defaultdict(dict)

    for expected_version, pdf_filename in VERSION_PDF_MAP.items():
        pdf_path = Path(pdf_dir) / pdf_filename
        if not pdf_path.exists():
            print(f"Warning: PDF file not found: {pdf_path}", file=sys.stderr)
            continue

        # Dynamically detect version from PDF (primary method)
        detected_version = identify_frame_version(pdf_path)

        if detected_version is None:
            print(f"ERROR: Could not identify version from {pdf_filename}", file=sys.stderr)
            print(f"  Expected version {expected_version} from VERSION_PDF_MAP", file=sys.stderr)
            print(f"  Skipping this PDF", file=sys.stderr)
            continue

        # Validate against expected version
        if detected_version != expected_version:
            print(f"WARNING: Version mismatch for {pdf_filename}:", file=sys.stderr)
            print(f"  Detected version: {detected_version}", file=sys.stderr)
            print(f"  Expected version: {expected_version} (from VERSION_PDF_MAP)", file=sys.stderr)
            print(f"  Using detected version {detected_version}", file=sys.stderr)

        version = detected_version  # Use detected version

        # Extract expected structures from List of Tables
        expected_structures = extract_expected_structures(pdf_path)
        print(f"Scanning {pdf_filename} (version {version})...", file=sys.stderr)
        print(f"  Expected {len(expected_structures)} structures from List of Tables", file=sys.stderr)

        # Extract each structure
        found_structures = []
        for object_name in expected_structures:
            # Skip structures that were removed in this version
            if (version, object_name) in PDF_REMOVED_STRUCTURES:
                print(f"  Skipping {object_name} (removed in version {version})", file=sys.stderr)
                continue

            structure = find_structure_in_pdf(pdf_path, object_name, version=version)
            if structure and structure['fields']:
                # Apply known PDF typo corrections
                # Track field names to handle duplicates
                field_name_counts = {}
                for field in structure['fields']:
                    field_name = field['name']

                    # Track how many times we've seen this field name
                    field_name_counts[field_name] = field_name_counts.get(field_name, 0) + 1
                    occurrence = field_name_counts[field_name]

                    # Check for correction key with occurrence number
                    correction_key = (version, object_name, field_name, occurrence)
                    if correction_key in PDF_TYPO_CORRECTIONS:
                        old_name = field['name']
                        new_name = PDF_TYPO_CORRECTIONS[correction_key]
                        field['name'] = new_name
                        print(f"  Corrected typo: {object_name}.{old_name}#{occurrence} -> {new_name}", file=sys.stderr)
                    else:
                        # Try without occurrence number for backward compatibility
                        correction_key = (version, object_name, field_name)
                        if correction_key in PDF_TYPO_CORRECTIONS:
                            old_name = field['name']
                            new_name = PDF_TYPO_CORRECTIONS[correction_key]
                            field['name'] = new_name
                            print(f"  Corrected typo: {object_name}.{old_name} -> {new_name}", file=sys.stderr)

                # Apply known PDF type corrections
                for field in structure['fields']:
                    type_correction_key = (version, object_name, field['name'])
                    if type_correction_key in PDF_TYPE_CORRECTIONS:
                        old_type = field['type']
                        new_type = PDF_TYPE_CORRECTIONS[type_correction_key]
                        field['type'] = new_type
                        print(f"  Corrected type: {object_name}.{field['name']}: {old_type} -> {new_type}", file=sys.stderr)

                # Apply known PDF comment corrections
                for field in structure['fields']:
                    comment_correction_key = (version, object_name, field['name'])
                    if comment_correction_key in PDF_COMMENT_CORRECTIONS:
                        old_comment = field.get('comment', '')
                        new_comment = PDF_COMMENT_CORRECTIONS[comment_correction_key]
                        field['comment'] = new_comment
                        print(f"  Corrected comment: {object_name}.{field['name']}: '{old_comment}' -> '{new_comment}'", file=sys.stderr)

                # Remove bogus fields created by parser errors
                fields_to_remove = []
                for i, field in enumerate(structure['fields']):
                    remove_key = (version, object_name, field['name'])
                    if remove_key in PDF_REMOVE_FIELDS:
                        fields_to_remove.append(i)
                        print(f"  Removing bogus field: {object_name}.{field['name']}", file=sys.stderr)
                # Remove in reverse order to preserve indices
                for i in reversed(fields_to_remove):
                    del structure['fields'][i]

                # Insert missing fields that were skipped by the parser
                for i, field in enumerate(structure['fields']):
                    missing_key = (version, object_name, field['name'])
                    if missing_key in PDF_MISSING_FIELDS:
                        field_name, field_type, field_desc = PDF_MISSING_FIELDS[missing_key]
                        new_field = {
                            'name': field_name,
                            'type': field_type,
                            'comment': field_desc
                        }
                        # Insert after the current field
                        structure['fields'].insert(i + 1, new_field)
                        print(f"  Inserted missing field: {object_name}.{field_name} ({field_type}) after {field['name']}", file=sys.stderr)
                        break  # Only insert once per structure

                # Final normalization pass: wrap any remaining raw pointers (for Version 9)
                # This must be done AFTER all corrections to avoid double-wrapping
                for field in structure['fields']:
                    field['type'] = normalize_type(field['type'], wrap_pointers=True)

                all_definitions[version][object_name] = structure
                found_structures.append(object_name)
                print(f"  Found {object_name}: {len(structure['fields'])} fields", file=sys.stderr)

        # Report any missing structures
        missing_structures = set(expected_structures) - set(found_structures)
        if missing_structures:
            print(f"  WARNING: Missing {len(missing_structures)} structures: {', '.join(sorted(missing_structures))}", file=sys.stderr)

    return all_definitions


def generate_cpp_source(definitions, output_file):
    """Generate C++ source file with hard-coded definitions."""
    with open(output_file, 'w') as f:
        f.write("""//
// LDASTools frameCPP - PDF-derived structure definitions
//
// Copyright (C) 2025 California Institute of Technology
//
// AUTO-GENERATED FILE - DO NOT EDIT
// Generated by generate_pdf_definitions.py
//

#include "PDFDefinitions.hh"

namespace FrameCPP {
namespace PDFDefinitions {

""")

        # Generate one function per version to avoid large function warnings
        for version in sorted(definitions.keys()):
            f.write(f"// Definitions for Version {version}\n")
            f.write(f"static void addDefinitionsV{version}(DefinitionMap& defs) {{\n")

            for object_name in sorted(definitions[version].keys()):
                struct = definitions[version][object_name]

                header_comment = escape_cpp_string(struct['header']['comment']) if struct['header'] else ""

                f.write(f"    // {object_name}\n")
                f.write(f"    defs[{version}][\"{object_name}\"] = {{\n")
                f.write(f"        \"{object_name}\",\n")
                f.write(f"        \"{header_comment}\",\n")
                f.write(f"        {{\n")

                for field in struct['fields']:
                    name = escape_cpp_string(field['name'])
                    type_str = escape_cpp_string(field['type'])
                    comment = escape_cpp_string(field['comment'])
                    f.write(f"            {{\"{name}\", \"{type_str}\", \"{comment}\"}},\n")

                f.write(f"        }}\n")
                f.write(f"    }};\n")

            f.write(f"}}\n\n")

        # Create main function that calls all version functions
        f.write("""static DefinitionMap createDefinitions() {
    DefinitionMap defs;

""")
        for version in sorted(definitions.keys()):
            f.write(f"    addDefinitionsV{version}(defs);\n")

        f.write("""
    return defs;
}

const DefinitionMap& getDefinitions() {
    static DefinitionMap defs = createDefinitions();
    return defs;
}

} // namespace PDFDefinitions
} // namespace FrameCPP
""")

    print(f"Generated {output_file}", file=sys.stderr)


def generate_cpp_header(output_file):
    """Generate C++ header file."""
    with open(output_file, 'w') as f:
        f.write("""//
// LDASTools frameCPP - PDF-derived structure definitions
//
// Copyright (C) 2025 California Institute of Technology
//
// AUTO-GENERATED FILE - DO NOT EDIT
// Generated by generate_pdf_definitions.py
//

#ifndef FRAMECPP_PDF_DEFINITIONS_HH
#define FRAMECPP_PDF_DEFINITIONS_HH

#include <map>
#include <string>
#include <vector>

namespace FrameCPP {
namespace PDFDefinitions {

struct FieldDef {
    std::string name;
    std::string type;
    std::string comment;
};

struct StructureDef {
    std::string name;
    std::string comment;
    std::vector<FieldDef> fields;
};

// Map: [version][object_name] -> StructureDef
using DefinitionMap = std::map<unsigned int, std::map<std::string, StructureDef>>;

const DefinitionMap& getDefinitions();

} // namespace PDFDefinitions
} // namespace FrameCPP

#endif // FRAMECPP_PDF_DEFINITIONS_HH
""")

    print(f"Generated {output_file}", file=sys.stderr)


def main():
    parser = argparse.ArgumentParser(
        description="Scan PDF specifications and generate C++ source with hard-coded definitions"
    )
    parser.add_argument("-d", "--pdf-dir", type=str,
                        default=os.path.expanduser("~/tmp/FrameSpecs"),
                        help="Directory containing PDF specification files")
    parser.add_argument("-o", "--output", type=str,
                        default="PDFDefinitions.cc",
                        help="Output C++ source file")
    parser.add_argument("--header", type=str,
                        default="PDFDefinitions.hh",
                        help="Output C++ header file")

    args = parser.parse_args()

    # Scan all PDFs
    definitions = scan_all_pdfs(args.pdf_dir)

    if not definitions:
        print("Error: No definitions found in PDFs", file=sys.stderr)
        sys.exit(1)

    # Generate C++ files
    generate_cpp_header(args.header)
    generate_cpp_source(definitions, args.output)

    print(f"\nSuccess! Generated {len(definitions)} versions with definitions.", file=sys.stderr)
    for version in sorted(definitions.keys()):
        print(f"  Version {version}: {len(definitions[version])} objects", file=sys.stderr)


if __name__ == "__main__":
    main()
