You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
48 lines
1.8 KiB
48 lines
1.8 KiB
from typing import List
|
|
from langchain.document_loaders.unstructured import UnstructuredFileLoader
|
|
import tqdm
|
|
|
|
|
|
class RapidOCRPDFLoader(UnstructuredFileLoader):
|
|
def _get_elements(self) -> List:
|
|
def pdf2text(filepath):
|
|
import fitz # pyMuPDF里面的fitz包，不要与pip install fitz混淆
|
|
from rapidocr_onnxruntime import RapidOCR
|
|
import numpy as np
|
|
ocr = RapidOCR()
|
|
doc = fitz.open(filepath)
|
|
resp = ""
|
|
|
|
b_unit = tqdm.tqdm(total=doc.page_count, desc="RapidOCRPDFLoader context page index: 0")
|
|
for i, page in enumerate(doc):
|
|
|
|
# 更新描述
|
|
b_unit.set_description("RapidOCRPDFLoader context page index: {}".format(i))
|
|
# 立即显示进度条更新结果
|
|
b_unit.refresh()
|
|
# TODO: 依据文本与图片顺序调整处理方式
|
|
text = page.get_text("")
|
|
resp += text + "\n"
|
|
|
|
img_list = page.get_images()
|
|
for img in img_list:
|
|
pix = fitz.Pixmap(doc, img[0])
|
|
img_array = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.width, -1)
|
|
result, _ = ocr(img_array)
|
|
if result:
|
|
ocr_result = [line[1] for line in result]
|
|
resp += "\n".join(ocr_result)
|
|
|
|
# 更新进度
|
|
b_unit.update(1)
|
|
return resp
|
|
|
|
text = pdf2text(self.file_path)
|
|
from unstructured.partition.text import partition_text
|
|
return partition_text(text=text, **self.unstructured_kwargs)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
loader = RapidOCRPDFLoader(file_path="../tests/samples/ocr_test.pdf")
|
|
docs = loader.load()
|
|
print(docs)
|
|
|