Update ppt_chunker.py

ppt_chunker.py

@@ -1,6 +1,5 @@
 from io import StringIO
 from typing import List
-from multiprocessing import Pool, cpu_count
 
 from unstructured.partition.pptx import partition_pptx
 from unstructured.cleaners.core import clean_ordered_bullets, clean_bullets, clean_non_ascii_chars
@@ -10,6 +9,34 @@ from ordered_multimap import OrderedMultiIndexMapWeakRef
 WRONG_NOUNS = {"BY", "IN", "ON", "INTO", "A", "AT", "WITH", "FROM", "TO", "AND", "OR", "BUT", "OWN", 'BEHIND', 'THROUGH', 'FIERCE', "'S"}
 NON_ENDING_PUNCT = {',', ':', ';', "'", '/', '-'}
 
+def process_chunk(chunk, nlp):
+    marked = []
+        
+    for i in range(len(chunk[1])):
+        current = chunk[1][i]
+            
+        if (type(current) is list) and current[1].isupper() and (current[0] == ('Title' or 'UncategorizedText')):
+            tokens = nlp(current[1])
+            
+            try:
+                next_ = chunk[1][i+1]
+                    
+                if type(next_) is not list:
+                    continue
+                
+            except IndexError:
+                continue
+                    
+            if next_[1].isupper() and (next_[0] == ('Title' or 'UncategorizedText')):
+                if (tokens[-1].pos_ in {'SYM', "ADP", 'ADV', 'PART', 'PRON', 'DET', "AUX", 'SCONJ', 'CONJ', "CCONJ"}) or ((tokens[-1].pos_ in {'PROPN', 'NOUN', 'VERB'}) and (str(tokens[-1]) in WRONG_NOUNS)):
+                    chunk[1][i+1][1] = current[1] + ' ' + next_[1]
+                    marked.append(i)
+
+    for i in marked:
+        del chunk[1][i]
+            
+    return chunk
+
 def ppt_chunk(file_like, nlp):
     import time
 
@@ -42,7 +69,7 @@ def ppt_chunk(file_like, nlp):
                 chunks.append([elem.id, current_chunk])
                 current_chunk = []
         else:
-            if elem.text[-1] in NON_ENDING_PUNCT:
+            if (elem.text[-1] in NON_ENDING_PUNCT) and (elem.category != 'Table'):
                 try:
                     next_ = elements[i+1]
                 except IndexError:
@@ -57,43 +84,29 @@ def ppt_chunk(file_like, nlp):
 
     sr = time.time()
 
-    def process_chunk(chunk):
-        marked = []
+    for chunk in chunks:
+        chunk = process_chunk(chunks, nlp)
+
+    while i < len(chunks):       
+        new_sub_chunks = []
+        only_tables = True
         
-        for i in range(len(chunk[1])):
-            current = chunk[1][i]
-            
-            if (type(current) is list) and current[1].isupper() and (current[0] == ('Title' or 'UncategorizedText')):
-                tokens = nlp(current[1])
-            
-                try:
-                    next_ = chunk[1][i+1]
-                    
-                    if type(next_) is not list:
-                        continue
+        for sub_chunk in chunk[1]:
+            if (type(sub_chunk) is list) and (sub_chunk[0] == 'Table'):
+                tables.append([chunk[0], sub_chunk])
+            else:
+                new_sub_chunks.append(sub_chunk)
+                only_tables = False
                 
-                except IndexError:
-                    continue
-                    
-                if next_[1].isupper() and (next_[0] == ('Title' or 'UncategorizedText')):
-                    if (tokens[-1].pos_ in {'SYM', "ADP", 'ADV', 'PART', 'PRON', 'DET', "AUX", 'SCONJ', 'CONJ', "CCONJ"}) or ((tokens[-1].pos_ in {'PROPN', 'NOUN', 'VERB'}) and (str(tokens[-1]) in WRONG_NOUNS)):
-                        chunk[1][i+1][1] = current[1] + ' ' + next_[1]
-                        marked.append(i)
-
-        for i in marked:
-            del chunk[1][i]
-            
-        return chunk
-
-    with Pool(cpu_count()) as pool:
-        chunks = pool.map(process_chunk, chunks)
+        if only_tables:
+            del chunks[i]
+        else:
+            chunks[i] = [chunk[0], new_sub_chunks]
+            i += 1
         
     er = time.time()
-    fr = er - sr
-    print(f'TIME aHERE {fr}')
-
-    for chunk in chunks:
-        print(f'CHUNK : {chunk}')
+    fr = er - s
+    print(f'TIME FINAL {fr}')
         
     weakDict = OrderedMultiIndexMapWeakRef()