Spaces:
Running
on
T4
Running
on
T4
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| import copy | |
| import re | |
| from io import BytesIO | |
| from PIL import Image | |
| from rag.nlp import tokenize, is_english | |
| from rag.nlp import rag_tokenizer | |
| from ppt_parser import PptParser | |
| class Ppt(PptParser): | |
| def __call__(self, fnm, from_page, to_page, callback=None): | |
| txts = super().__call__(fnm, from_page, to_page) | |
| callback(0.5, "Text extraction finished.") | |
| import aspose.slides as slides | |
| import aspose.pydrawing as drawing | |
| imgs = [] | |
| with slides.Presentation(BytesIO(fnm)) as presentation: | |
| for i, slide in enumerate(presentation.slides[from_page: to_page]): | |
| buffered = BytesIO() | |
| slide.get_thumbnail( | |
| 0.5, 0.5).save( | |
| buffered, drawing.imaging.ImageFormat.jpeg) | |
| imgs.append(Image.open(buffered)) | |
| assert len(imgs) == len( | |
| txts), "Slides text and image do not match: {} vs. {}".format(len(imgs), len(txts)) | |
| callback(0.9, "Image extraction finished") | |
| self.is_english = is_english(txts) | |
| return [(txts[i], imgs[i]) for i in range(len(txts))] | |
| def chunk(filename, binary=None, from_page=0, to_page=100000, | |
| lang="English", callback=None, **kwargs): | |
| """ | |
| The supported file formats are pptx. | |
| Every page will be treated as a chunk. And the thumbnail of every page will be stored. | |
| PPT file will be parsed by using this method automatically, setting-up for every PPT file is not necessary. | |
| """ | |
| eng = lang.lower() == "english" | |
| doc = { | |
| "docnm_kwd": filename, | |
| "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename)) | |
| } | |
| doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"]) | |
| res = [] | |
| ppt_parser = Ppt() | |
| for pn, (txt, img) in enumerate( | |
| ppt_parser(filename if not binary else binary, from_page, 1000000, callback) | |
| ): | |
| d = copy.deepcopy(doc) | |
| pn += from_page | |
| d["image"] = img | |
| d["page_num_int"] = [pn + 1] | |
| d["top_int"] = [0] | |
| d["position_int"] = [(pn + 1, 0, img.size[0], 0, img.size[1])] | |
| tokenize(d, txt, eng) | |
| res.append(d) | |
| return res | |
| raise NotImplementedError( | |
| "file type not supported yet(pptx, pdf supported)") |