Erstellen eines benutzerdefinierten PDF -Parsers mit PYPDF und Langchain

custom_pdf_parser/
│
├── parser.py           
├── langchain_loader.py  
├── pipeline.py          
├── instance.py     
├── necessities.txt     # Dependencies record
└── __init__.py         # (Non-compulsory) to mark listing as Python bundle

pip set up pypdf langchain

pip set up -r necessities.txt

import os
import logging
from pathlib import Path
from typing import Listing, Dict, Any
import pypdf
from pypdf import PdfReader
# Configure logging to point out data and above messages
logging.basicConfig(stage=logging.INFO)
logger = logging.getLogger(__name__)
class CustomPDFParser:
  def __init__(
      self,extract_images: bool = False,preserve_layout: bool = True,remove_headers_footers: bool = True,min_text_length: int = 10
  ):
      """
      Initialize the parser with choices to extract photographs, protect structure, take away repeated headers/footers, and minimal textual content size for pages.
      Args:
          extract_images: Whether or not to extract picture data from pages
          preserve_layout: Whether or not to maintain structure spacing in textual content extraction
          remove_headers_footers: Whether or not to detect and take away headers/footers
          min_text_length: Minimal size of textual content for a web page to be thought of legitimate
      """
      self.extract_images = extract_images
      self.preserve_layout = preserve_layout
      self.remove_headers_footers = remove_headers_footers
      self.min_text_length = min_text_length
  def extract_text_from_page(self, web page: pypdf.PageObject, page_num: int) -> Dict(str, Any):
      """
      Extract textual content and metadata from a single PDF web page.
      Args:
          web page: PyPDF web page object
          page_num: zero-based web page quantity
      Returns:
          dict with keys:
              - 'textual content': extracted and cleaned textual content string,
              - 'metadata': web page metadata dict,
              - 'word_count': variety of phrases in extracted textual content
      """
      attempt:
 # Extract textual content, optionally preserving the structure for higher formatting
          if self.preserve_layout:
              textual content = web page.extract_text(extraction_mode="structure")
          else:
              textual content = web page.extract_text()
        # Clear textual content: take away additional whitespace and normalize paragraphs
          textual content = self._clean_text(textual content)
        # Collect web page metadata (web page quantity, rotation angle, mediabox)
          metadata = {
              "page_number": page_num + 1,  # 1-based numbering
              "rotation": getattr(web page, "rotation", 0),
              "mediabox": str(getattr(web page, "mediabox", None)),
          }
          # Optionally, extract picture data from web page if requested
          if self.extract_images:
              metadata("photographs") = self._extract_image_info(web page)
          # Return dictionary with textual content and metadata for this web page
          return {
              "textual content": textual content,
              "metadata": metadata,
              "word_count": len(textual content.break up()) if textual content else 0
          }
      besides Exception as e:
          # Log error and return empty information for problematic pages
          logger.error(f"Error extracting web page {page_num}: {e}")
          return {
              "textual content": "",
              "metadata": {"page_number": page_num + 1, "error": str(e)},
              "word_count": 0
          }
  def _clean_text(self, textual content: str) -> str:
      """
      Clear and normalize extracted textual content, preserving paragraph breaks.
      Args:
          textual content: uncooked textual content extracted from PDF web page
      Returns:
          cleaned textual content string
      """
      if not textual content:
          return ""
      traces = textual content.break up('n')
      cleaned_lines = ()
      for line in traces:
          line = line.strip()  # Take away main/trailing whitespace
          if line:
              # Non-empty line; hold it
              cleaned_lines.append(line)
          elif cleaned_lines and cleaned_lines(-1):
              # Protect paragraph break by preserving empty line provided that earlier line exists
              cleaned_lines.append("")
      cleaned_text="n".be a part of(cleaned_lines)
#Cut back any situations of greater than two consecutive clean traces to 2
      whereas 'nnn' in cleaned_text:
          cleaned_text = cleaned_text.exchange('nnn', 'nn')
      return cleaned_text.strip()
  def _extract_image_info(self, web page: pypdf.PageObject) -> Listing(Dict(str, Any)):
      """
      Extract fundamental picture metadata from web page, if obtainable.
      Args:
          web page: PyPDF web page object
      Returns:
          Listing of dictionaries with picture data (index, identify, width, top)
      """
      photographs = ()
      attempt:
          # PyPDF pages can have an 'photographs' attribute itemizing embedded photographs
          if hasattr(web page, 'photographs'):
              for i, picture in enumerate(web page.photographs):
                  photographs.append({
                      "image_index": i,
                      "identify": getattr(picture, 'identify', f"image_{i}"),
                      "width": getattr(picture, 'width', None),
                      "top": getattr(picture, 'top', None)
                  })
      besides Exception as e:
          logger.warning(f"Picture extraction failed: {e}")
      return photographs

  def _remove_headers_footers(self, pages_data: Listing(Dict(str, Any))) -> Listing(Dict(str, Any)):
      """
      Take away repeated headers and footers that seem on many pages.
      That is accomplished by figuring out traces showing on over 50% of pages
      in the beginning or finish of web page textual content, then eradicating these traces.
      Args:
          pages_data: Listing of dictionaries representing every web page's extracted information.
      Returns:
          Up to date record of pages with headers/footers eliminated
      """
      # Solely try elimination if sufficient pages and possibility enabled
      if len(pages_data) < 3 or not self.remove_headers_footers:
          return pages_data
      # Acquire first and final traces from every web page's textual content for evaluation
      first_lines = (web page("textual content").break up('n')(0) if web page("textual content") else "" for web page in pages_data)
      last_lines = (web page("textual content").break up('n')(-1) if web page("textual content") else "" for web page in pages_data)
      threshold = len(pages_data) * 0.5  # Greater than 50% pages
      # Establish candidate headers and footers showing continuously
      potential_headers = (line for line in set(first_lines)
                          if first_lines.rely(line) > threshold and line.strip())
      potential_footers = (line for line in set(last_lines)
                          if last_lines.rely(line) > threshold and line.strip())
      # Take away recognized headers and footers from every web page's textual content
      for page_data in pages_data:
          traces = page_data("textual content").break up('n')
          # Take away header if it matches a frequent header
          if traces and potential_headers:
              for header in potential_headers:
                  if traces(0).strip() == header.strip():
                      traces = traces(1:)
                      break
          # Take away footer if it matches a frequent footer
          if traces and potential_footers:
              for footer in potential_footers:
                  if traces(-1).strip() == footer.strip():
                      traces = traces(:-1)
                      break

          page_data("textual content") = 'n'.be a part of(traces).strip()
      return pages_data
  def _extract_document_metadata(self, pdf_reader: PdfReader, pdf_path: str) -> Dict(str, Any):
      """
      Extract metadata from the PDF doc itself.
      Args:
          pdf_reader: PyPDF PdfReader occasion
          pdf_path: path to PDF file
      Returns:
          Dictionary of metadata together with file data and PDF doc metadata
      """
      metadata = {
          "file_path": pdf_path,
          "file_name": Path(pdf_path).identify,
          "file_size": os.path.getsize(pdf_path) if os.path.exists(pdf_path) else None,
      }
      attempt:
          if pdf_reader.metadata:
              # Extract widespread PDF metadata keys if obtainable
              metadata.replace({
                  "title": pdf_reader.metadata.get('/Title', ''),
                  "writer": pdf_reader.metadata.get('/Writer', ''),
                  "topic": pdf_reader.metadata.get('/Topic', ''),
                  "creator": pdf_reader.metadata.get('/Creator', ''),
                  "producer": pdf_reader.metadata.get('/Producer', ''),
                  "creation_date": str(pdf_reader.metadata.get('/CreationDate', '')),
                  "modification_date": str(pdf_reader.metadata.get('/ModDate', '')),
              })
      besides Exception as e:
          logger.warning(f"Metadata extraction failed: {e}")
      return metadata
  def parse_pdf(self, pdf_path: str) -> Dict(str, Any):
      """
      Parse your entire PDF file. Opens the file, extracts textual content and metadata web page by web page, removes headers/footers if configured, and aggregates outcomes.
      Args:
          pdf_path: Path to the PDF file
      Returns:
          Dictionary with keys:
              - 'full_text': mixed textual content from all pages,
              - 'pages': record of page-wise dicts with textual content and metadata,
              - 'document_metadata': file and PDF metadata,
              - 'total_pages': complete pages in PDF,
              - 'processed_pages': variety of pages stored after filtering,
              - 'total_words': complete phrase rely of parsed textual content
      """
      attempt:
          with open(pdf_path, 'rb') as file:
              pdf_reader = PdfReader(file)
              doc_metadata = self._extract_document_metadata(pdf_reader, pdf_path)
              pages_data = ()
              # Iterate over all pages and extract information
              for i, web page in enumerate(pdf_reader.pages):
                  page_data = self.extract_text_from_page(web page, i)
                  # Solely hold pages with adequate textual content size
                  if len(page_data("textual content")) >= self.min_text_length:
                      pages_data.append(page_data)
              # Take away repeated headers and footers
              pages_data = self._remove_headers_footers(pages_data)
           # Mix all web page texts with a double newline as a separator
              full_text="nn".be a part of(web page("textual content") for web page in pages_data if web page("textual content"))
              # Return closing structured information
              return {
                  "full_text": full_text,
                  "pages": pages_data,
                  "document_metadata": doc_metadata,
                  "total_pages": len(pdf_reader.pages),
                  "processed_pages": len(pages_data),
                  "total_words": sum(web page("word_count") for web page in pages_data)
              }
      besides Exception as e:
          logger.error(f"Did not parse PDF {pdf_path}: {e}")
          increase

from typing import Listing, Non-compulsory, Dict, Any
from langchain.schema import Doc
from langchain.document_loaders.base import BaseLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from parser import CustomPDFParser  # import the parser outlined above
class LangChainPDFLoader(BaseLoader):
   def __init__(
       self,file_path: str,parser_config: Non-compulsory(Dict(str, Any)) = None,chunk_size: int = 500, chunk_overlap: int = 50
   ):
       """
       Initialize the loader with the PDF file path, parser configuration, and chunking parameters.
       Args:
           file_path: path to PDF file
           parser_config: dictionary of parser choices
           chunk_size: chunk dimension for splitting lengthy texts
           chunk_overlap: chunk overlap for splitting
       """
       self.file_path = file_path
       self.parser_config = parser_config or {}
       self.chunk_size = chunk_size
       self.chunk_overlap = chunk_overlap
       self.parser = CustomPDFParser(**self.parser_config)
   def load(self) -> Listing(Doc):
       """
       Load PDF, parse pages, and convert every web page to a LangChain Doc.
       Returns:
           Listing of Doc objects with web page textual content and mixed metadata.
       """
       parsed_data = self.parser.parse_pdf(self.file_path)
       paperwork = ()
       # Convert every web page dict to a LangChain Doc
       for page_data in parsed_data("pages"):
           if page_data("textual content"):
               # Merge document-level and page-level metadata
               metadata = {**parsed_data("document_metadata"), **page_data("metadata")}
               doc = Doc(page_content=page_data("textual content"), metadata=metadata)
               paperwork.append(doc)
       return paperwork
   def load_and_split(self) -> Listing(Doc):
       """
       Load the PDF and break up massive paperwork into smaller chunks.
       Returns:
           Listing of Doc objects after splitting massive texts.
       """
       paperwork = self.load()
       # Initialize a textual content splitter with the specified chunk dimension and overlap
       text_splitter = RecursiveCharacterTextSplitter(
           chunk_size=self.chunk_size,
           chunk_overlap=self.chunk_overlap,
           separators=("nn", "n", " ", "")  # hierarchical splitting
       )
       # Cut up paperwork into smaller chunks
       split_docs = text_splitter.split_documents(paperwork)
       return split_docs

from typing import Listing, Non-compulsory, Dict, Any
from langchain.schema import Doc
from parser import CustomPDFParser
from langchain_loader import LangChainPDFLoader
import logging
logger = logging.getLogger(__name__)
class PDFProcessingPipeline:
   def __init__(self, parser_config: Non-compulsory(Dict(str, Any)) = None):
       """
       Args:
          parser_config: dictionary of choices handed to CustomPDFParser
       """
       self.parser_config = parser_config or {}
   def process_single_pdf(
       self,pdf_path: str,output_format: str = "langchain",chunk_documents: bool = True,chunk_size: int = 500,chunk_overlap: int = 50
   ) -> Any:
       """
       Args:
           pdf_path: path to PDF file
           output_format: "uncooked" (dict), "langchain" (Paperwork), or "textual content" (string)
           chunk_documents: whether or not to separate LangChain paperwork into chunks
           chunk_size: chunk dimension for splitting
           chunk_overlap: chunk overlap for splitting
       Returns:
           Parsed content material within the requested format
       """
       if output_format == "uncooked":
           # Use uncooked CustomPDFParser output
           parser = CustomPDFParser(**self.parser_config)
           return parser.parse_pdf(pdf_path)
       elif output_format == "langchain":
           # Use LangChain loader, optionally chunked
           loader = LangChainPDFLoader(pdf_path, self.parser_config, chunk_size, chunk_overlap)
           if chunk_documents:
               return loader.load_and_split()
           else:
               return loader.load()
       elif output_format == "textual content":
           # Return mixed plain textual content solely
           parser = CustomPDFParser(**self.parser_config)
           parsed_data = parser.parse_pdf(pdf_path)
           return parsed_data.get("full_text", "")
       else:
           increase ValueError(f"Unknown output_format: {output_format}")

import os
from pathlib import Path
def principal():
   print("👋 Welcome to the Customized PDF Parser!")
   print("What would you love to do?")
   print("1. View full parsed uncooked information")
   print("2. Extract full plain textual content")
   print("3. Get LangChain paperwork (no chunking)")
   print("4. Get LangChain paperwork (with chunking)")
   print("5. Present doc metadata")
   print("6. Present per-page metadata")
   print("7. Present cleaned web page textual content (header/footer eliminated)")
   print("8. Present extracted picture metadata")
   alternative = enter("Enter the variety of your alternative: ").strip()
   if alternative not in {'1', '2', '3', '4', '5', '6', '7', '8'}:
       print("❌ Invalid possibility.")
       return
   file_path = enter("Enter the trail to your PDF file: ").strip()
   if not Path(file_path).exists():
       print("❌ File not discovered.")
       return
   # Initialize pipeline
   pipeline = PDFProcessingPipeline({
       "preserve_layout": False,
       "remove_headers_footers": True,
       "extract_images": True,
       "min_text_length": 20
   })
   # Uncooked information is required for many choices
   parsed = pipeline.process_single_pdf(file_path, output_format="uncooked")
   if alternative == '1':
       print("nFull Uncooked Parsed Output:")
       for okay, v in parsed.objects():
           print(f"{okay}: {str(v)(:300)}...")
   elif alternative == '2':
       print("nFull Cleaned Textual content (truncated preview):")
       print("Previewing the primary 1000 characters:n"+parsed("full_text")(:1000), "...")
   elif alternative == '3':
       docs = pipeline.process_single_pdf(file_path, output_format="langchain", chunk_documents=False)
       print(f"nLangChain Paperwork: {len(docs)}")
       print("Previewing the primary 500 characters:n", docs(0).page_content(:500), "...")
   elif alternative == '4':
       docs = pipeline.process_single_pdf(file_path, output_format="langchain", chunk_documents=True)
       print(f"nLangChain Chunks: {len(docs)}")
       print("Pattern chunk content material (first 500 chars):")
       print(docs(0).page_content(:500), "...")
   elif alternative == '5':
       print("nDocument Metadata:")
       for key, worth in parsed("document_metadata").objects():
           print(f"{key}: {worth}")
   elif alternative == '6':
       print("nPer-page Metadata:")
       for i, web page in enumerate(parsed("pages")):
           print(f"Web page {i+1}: {web page('metadata')}")
   elif alternative == '7':
       print("nCleaned Textual content After Header/Footer Removing.")
       print("Exhibiting the primary 3 pages and first 500 characters of the textual content from every web page.")
       for i, web page in enumerate(parsed("pages")(:3)):  # First 3 pages
           print(f"n--- Web page {i+1} ---")
           print(web page("textual content")(:500), "...")
   elif alternative == '8':
       print("nExtracted Picture Metadata (if obtainable):")
       discovered = False
       for i, web page in enumerate(parsed("pages")):
           photographs = web page("metadata").get("photographs", ())
           if photographs:
               discovered = True
               print(f"n--- Web page {i+1} ---")
               for img in photographs:
                   print(img)
       if not discovered:
           print("No picture metadata discovered.")
if __name__ == "__main__":
   principal()

👋 Welcome to the Customized PDF Parser!
What would you love to do?
1. View full parsed uncooked information
2. Extract full plain textual content
3. Get LangChain paperwork (no chunking)
4. Get LangChain paperwork (with chunking)
5. Present doc metadata
6. Present per-page metadata
7. Present cleaned web page textual content (header/footer eliminated)
8. Present extracted picture metadata.
Enter the variety of your alternative: 5
Enter the trail to your PDF file: /content material/articles.pdf

Output:
LangChain Chunks: 16
First chunk preview:
San José State College Writing Heart
www.sjsu.edu/writingcenter
Written by Ben Aldridge

Articles (a/an/the), Spring 2014.                                                                                   1 of 4
Articles (a/an/the)

There are three articles within the English language: a, an, and the. They're positioned earlier than nouns
and present whether or not a given noun is common or particular.

Examples of Articles