new_version

.gitattributes

@@ -29,3 +29,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+appStore/img/giz_sdsn.jpg filter=lfs diff=lfs merge=lfs -text
+appStore/img/paris.png filter=lfs diff=lfs merge=lfs -text
+appStore/img/pic1.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,17 @@
+import appStore.keyword_search as keyword_search
+import appStore.sdg_analysis as sdg_analysis
+import appStore.coherence as coherence
+import appStore.info as info
+from appStore.multiapp import MultiApp
+import streamlit as st
+
+st.set_page_config(f'SDSN x GIZ Policy Action Tracking v0.1', layout="wide")
+
+app = MultiApp()
+
+app.add_app("Analyse Policy Document", sdg_analysis.app)
+app.add_app("KeyWord Search", keyword_search.app)
+app.add_app("Check Coherence", coherence.app)
+app.add_app("Info", info.app)
+
+app.run()

appStore/__init__.py

@@ -0,0 +1 @@
+# creating appstore package

appStore/coherence.py

@@ -0,0 +1,231 @@
+# set path
+import glob, os, sys; sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+from sklearn.metrics.pairwise import cosine_similarity
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+import urllib.request
+import ast
+import tempfile
+import sqlite3
+import json
+import urllib.request
+import ast
+def app():
+    # Sidebar
+    st.sidebar.title('Check Coherence')
+    st.sidebar.write(' ')
+    with open('ndcs/countryList.txt') as dfile:
+        countryList = dfile.read()
+
+    countryList = ast.literal_eval(countryList)
+    countrynames = list(countryList.keys())
+    
+    option = st.sidebar.selectbox('Select Country', (countrynames))
+    countryCode = countryList[option]
+
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> Check Coherence of Policy Document with NDCs</h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=True):
+
+        st.write(
+            """     
+            The *Check Coherence* app is an easy-to-use interface built in Streamlit for doing analysis of policy document and finding the coherence between NDCs/New-Updated NDCs- developed by GIZ Data and the Sustainable Development Solution Network.
+            """
+        )
+
+        st.markdown("")
+
+    st.markdown("")
+    st.markdown("##  📌 Step One: Upload document of the country selected ")
+    
+    with st.container():
+            docs = None
+            # asking user for either upload or select existing doc
+            choice = st.radio(label = 'Select the Document',
+                              help = 'You can upload the document \
+                              or else you can try a example document.', 
+                              options = ('Upload Document', 'Try Example'), 
+                              horizontal = True)
+
+            if choice == 'Upload Document':
+              uploaded_file = st.file_uploader('Upload the File', type=['pdf', 'docx', 'txt'])
+              if uploaded_file is not None:
+                with tempfile.NamedTemporaryFile(mode="wb") as temp:
+                    bytes_data = uploaded_file.getvalue()
+                    temp.write(bytes_data)
+
+                    st.write("Uploaded Filename: ", uploaded_file.name)
+                    file_name =  uploaded_file.name
+                    file_path = temp.name
+                    docs = pre.load_document(file_path, file_name)
+                    haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+            else:
+              # listing the options
+              option = st.selectbox('Select the example document',
+                                    ('South Africa:Low Emission strategy', 
+                                    'Ethiopia: 10 Year Development Plan'))
+              if option is 'South Africa:Low Emission strategy':
+                file_name = file_path  = 'sample/South Africa_s Low Emission Development Strategy.txt'
+                countryCode = countryList['South Africa']
+                st.write("Selected document:", file_name.split('/')[1])
+                # with open('sample/South Africa_s Low Emission Development Strategy.txt') as dfile:
+                # file = open('sample/South Africa_s Low Emission Development Strategy.txt', 'wb')
+              else:
+                # with open('sample/Ethiopia_s_2021_10 Year Development Plan.txt') as dfile:
+                file_name = file_path =  'sample/Ethiopia_s_2021_10 Year Development Plan.txt'
+                countryCode = countryList['Ethiopia']
+                st.write("Selected document:", file_name.split('/')[1])
+              
+              if option is not None:
+                docs = pre.load_document(file_path,file_name)
+                haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+            with open('ndcs/cca.txt', encoding='utf-8', errors='ignore') as dfile:
+                cca_sent = dfile.read()
+
+            cca_sent = ast.literal_eval(cca_sent)
+            
+            with open('ndcs/ccm.txt', encoding='utf-8', errors='ignore') as dfile:
+                ccm_sent = dfile.read()
+
+            ccm_sent = ast.literal_eval(ccm_sent)
+            
+            with open('ndcs/countryList.txt') as dfile:
+                countryList = dfile.read()
+
+            countryList = ast.literal_eval(countryList)
+            
+            def get_document(countryCode: str):
+                link = "https://klimalog.die-gdi.de/ndc/open-data/dataset.json"  
+                with urllib.request.urlopen(link) as urlfile:
+                    data =  json.loads(urlfile.read())
+                categoriesData = {}
+                categoriesData['categories']= data['categories']
+                categoriesData['subcategories']= data['subcategories']
+                keys_sub = categoriesData['subcategories'].keys()
+                documentType= 'NDCs'
+                if documentType in data.keys():
+                    if countryCode in data[documentType].keys():
+                        get_dict = {}
+                        for key, value in data[documentType][countryCode].items():
+                            if key not in ['country_name','region_id', 'region_name']:
+                                get_dict[key] = value['classification']
+                            else:
+                                get_dict[key] = value
+                    else:
+                        return None
+                else:
+                    return None
+
+                country = {}
+                for key in categoriesData['categories']:
+                    country[key]= {}
+                for key,value in categoriesData['subcategories'].items():
+                    country[value['category']][key] = get_dict[key]
+                
+                return country
+        
+        #   country_ndc = get_document('NDCs', countryList[option])
+            
+            def countrySpecificCCA(cca_sent, threshold, countryCode):
+                temp = {}
+                doc = get_document(countryCode)
+                for key,value in cca_sent.items():
+                    id_ = doc['climate change adaptation'][key]['id']
+                    if id_ >threshold:
+                        temp[key] = value['id'][id_]
+                return temp
+            
+                
+            def countrySpecificCCM(ccm_sent, threshold, countryCode):
+                temp = {}
+                doc = get_document(countryCode)
+                for key,value in ccm_sent.items():
+                    id_ = doc['climate change mitigation'][key]['id']
+                    if id_ >threshold:
+                        temp[key] = value['id'][id_]
+                
+                return temp
+
+        
+        
+            if docs is not None:
+                    sent_cca = countrySpecificCCA(cca_sent,1,countryCode)
+                    sent_ccm = countrySpecificCCM(ccm_sent,1,countryCode)
+                    #st.write(sent_ccm)
+                    @st.cache(allow_output_mutation=True)
+                    def load_sentenceTransformer(name):
+                        return SentenceTransformer(name)
+                    model = load_sentenceTransformer('all-MiniLM-L6-v2')
+          
+                    document_embeddings = model.encode(paraList, show_progress_bar=True)
+                
+                    genre = st.radio( "Select Category",('Climate Change Adaptation', 'Climate Change Mitigation'))
+                    if genre == 'Climate Change Adaptation':
+                        sent_dict = sent_cca
+                        sent_labels = []
+                        for key,sent in sent_dict.items():
+                            sent_labels.append(sent)
+                        label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+                        similarity_high_threshold = 0.55
+                        similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                        label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                        positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+                        
+                        
+                    else:
+                        sent_dict = sent_ccm
+                        sent_labels = []
+                        for key,sent in sent_dict.items():
+                            sent_labels.append(sent)
+                        label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+                        similarity_high_threshold = 0.55
+                        similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                        label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                        positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+                        
+            
+                #    sent_labels = []
+                #   for key,sent in sent_dict.items():
+                  #      sent_labels.append(sent)
+                    
+            
+                  # label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+            
+                    #similarity_high_threshold = 0.55
+                  # similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                    #label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                    #positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+            
+                    for _label_idx, _paragraph_idx in positive_indices:
+                        st.write("This paragraph: \n")
+                        st.write(paraList[_paragraph_idx])
+                        st.write(f"Is relevant to: \n {list(sent_dict.keys())[_label_idx]}")
+                        st.write('-'*10)
+            

appStore/info.py

@@ -0,0 +1,47 @@
+import streamlit as st
+
+
+def app():
+    with open('style.css') as f:
+        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+    footer = """
+           <div class="footer-custom">
+               Developer - <a href="https://www.linkedin.com/in/erik-lehmann-giz/" target="_blank">Erik Lehmann</a>  |   
+               <a href="https://www.linkedin.com/in/jonas-nothnagel-bb42b114b/" target="_blank">Jonas Nothnagel</a>   |
+               <a href="https://www.linkedin.com/in/prashantpsingh/" target="_blank">Prashant Singh</a> |
+               Guidance & Feedback - Maren Bernlöhr | Manuel Kuhn </a>
+           </div>
+       """
+    st.markdown(footer, unsafe_allow_html=True)
+
+    st.subheader("Policy Action Tracker Manual")
+    intro = """
+    <div class="text">
+    The manual extraction of relevant information from text documents is a time-consuming task for any policy analyst. 
+    As the amount and length of public policy documents in relation to sustainable development (such as National Development Plans and 
+    Nationally Determined Contributions) continuously increases, a major challenge for policy action tracking – the evaluation of stated 
+    goals and targets and their actual implementation on the ground – arises. Luckily, Artificial Intelligence (AI) and Natural Language Processing (NLP) 
+    methods can help in shortening and easing this task for policy analysts. 
+    For this purpose, the United Nations Sustainable Development Solutions Network (SDSN) and the Deutsche Gesellschaft für Internationale Zusammenarbeit (GIZ) GmbH 
+    are collaborating since 2021 in the development of an AI-powered open-source web application that helps find and extract relevant information from public policy
+     documents faster to facilitate evidence-based decision-making processes in sustainable development and beyond. 
+        <ul>
+        <li>Analizing the policy document</li>
+        <li>finding SDG related content</li> 
+        <li>Make it searchable</li>
+        <li>compare it to the national NDC</li>
+        </ul> 
+    </div>
+    <br>
+    """
+    st.markdown(intro, unsafe_allow_html=True)
+    st.image("appStore/img/pic1.png", caption="NDC Coherence")
+    st.subheader("Methodology")
+    #st.write("Each sentence in the generated answer ends with a coloured tooltip; the colour ranges from red to green. "
+    #         "The tooltip contains a value representing answer sentence similarity to a specific sentence in the "
+    #         "Wikipedia context passages retrieved.  Mouseover on the tooltip will show the sentence from the "
+    #         "Wikipedia context passage. If a sentence similarity is 1.0, the seq2seq model extracted and "
+    #         "copied the sentence verbatim from Wikipedia context passages. Lower values of sentence "
+    #         "similarity indicate the seq2seq model is struggling to generate a relevant sentence for the question "
+    #         "asked.")
+    #st.image("wikipedia_answer.png", caption="Answer with similarity tooltips")

appStore/keyword_search.py

@@ -0,0 +1,504 @@
+# set path
+import glob, os, sys; sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+
+import tempfile
+import sqlite3
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center;  \
+                      color: black;'> Keyword Search</h1>", 
+                      unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=True):
+
+        st.write(
+            """     
+            The *Keyword Search* app is an easy-to-use interface \ 
+            built in Streamlit for doing keyword search in \
+            policy document - developed by GIZ Data and the \
+            Sustainable Development Solution Network.
+            """
+        )
+
+        st.markdown("")
+
+    st.markdown("")
+    st.markdown("### 📌 Step One: Upload document ### ")
+
+    with st.container():
+      def bm25_tokenizer(text):
+            tokenized_doc = []
+            for token in text.lower().split():
+                token = token.strip(string.punctuation)
+
+                if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
+                    tokenized_doc.append(token)
+            return tokenized_doc
+          
+      def bm25TokenizeDoc(paraList):
+          tokenized_corpus = []
+          for passage in tqdm(paraList):
+              if len(passage.split()) >256:
+                  temp  = " ".join(passage.split()[:256])
+                  tokenized_corpus.append(bm25_tokenizer(temp))
+                  temp  = " ".join(passage.split()[256:])
+                  tokenized_corpus.append(bm25_tokenizer(temp))
+              else:
+                  tokenized_corpus.append(bm25_tokenizer(passage))
+                  
+          return tokenized_corpus
+      def search(keyword):
+                ##### BM25 search (lexical search) #####
+                bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
+                top_n = np.argpartition(bm25_scores, -10)[-10:]
+                bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+                bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+                
+                ##### Sematic Search #####
+                # Encode the query using the bi-encoder and find potentially relevant passages
+                #query = "Does document contain {} issues ?".format(keyword)
+                question_embedding = bi_encoder.encode(keyword, convert_to_tensor=True)
+          
+                hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+                hits = hits[0]  # Get the hits for the first query
+                
+                
+                ##### Re-Ranking #####
+                # Now, score all retrieved passages with the cross_encoder
+                #cross_inp = [[query, paraList[hit['corpus_id']]] for hit in hits]
+                #cross_scores = cross_encoder.predict(cross_inp)
+                
+                # Sort results by the cross-encoder scores
+                #for idx in range(len(cross_scores)):
+                  #   hits[idx]['cross-score'] = cross_scores[idx]
+                  
+                
+                return bm25_hits, hits
+
+      def show_results(keywordList):
+        for keyword in keywordList:
+          st.write("Results for Query: {}".format(keyword))
+          bm25_hits, hits = search(keyword)     
+
+          st.markdown("""
+                      We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+                      """)  
+          # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+          st.markdown("Top few lexical search (BM25) hits")
+          for hit in bm25_hits[0:5]:
+              if hit['score'] > 0.00:   
+                  st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+          
+          
+        
+        
+        
+        #   st.table(bm25_hits[0:3])
+          
+          st.markdown("\n-------------------------\n")
+          st.markdown("Top few Bi-Encoder Retrieval hits")
+          
+          hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+          for hit in hits[0:5]:
+            #  if hit['score'] > 0.45:
+              st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+          #st.table(hits[0:3]
+
+
+      @st.cache(allow_output_mutation=True)
+      def load_sentenceTransformer(name):
+          return SentenceTransformer(name)
+
+
+
+      docs = None
+      # asking user for either upload or select existing doc
+      choice = st.radio(label = 'Select the Document',
+                        help = 'You can upload the document \
+                        or else you can try a example document', 
+                        options = ('Upload Document', 'Try Example'), 
+                        horizontal = True)
+
+      if choice == 'Upload Document':
+        uploaded_file = st.file_uploader('Upload the File', type=['pdf', 'docx', 'txt'])
+        if uploaded_file is not None:
+          with tempfile.NamedTemporaryFile(mode="wb") as temp:
+              bytes_data = uploaded_file.getvalue()
+              temp.write(bytes_data)
+
+              st.write("Uploaded Filename: ", uploaded_file.name)
+              file_name =  uploaded_file.name
+              file_path = temp.name
+              docs = pre.load_document(file_path, file_name)
+              haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+      else:
+        # listing the options
+        option = st.selectbox('Select the example document',
+                              ('South Africa:Low Emission strategy', 
+                              'Ethiopia: 10 Year Development Plan'))
+        if option is 'South Africa:Low Emission strategy':
+          file_name = file_path  = 'sample/South Africa_s Low Emission Development Strategy.txt'
+          st.write("Selected document:", file_name.split('/')[1])
+          # with open('sample/South Africa_s Low Emission Development Strategy.txt') as dfile:
+          # file = open('sample/South Africa_s Low Emission Development Strategy.txt', 'wb')
+        else:
+          # with open('sample/Ethiopia_s_2021_10 Year Development Plan.txt') as dfile:
+          file_name = file_path =  'sample/Ethiopia_s_2021_10 Year Development Plan.txt'
+          st.write("Selected document:", file_name.split('/')[1])
+        
+        if option is not None:
+          docs = pre.load_document(file_path,file_name)
+          haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+        if docs is not None:
+          
+          bi_encoder = load_sentenceTransformer('msmarco-distilbert-cos-v5') # multi-qa-MiniLM-L6-cos-v1
+          bi_encoder.max_seq_length = 64     #Truncate long passages to 256 tokens
+          top_k = 32
+
+          document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
+          tokenized_corpus = bm25TokenizeDoc(paraList)
+          document_bm25 = BM25Okapi(tokenized_corpus)
+          keywordList = None
+
+          col1, col2 = st.columns(2)
+          with col1:
+            if st.button('Climate Change Keyword Search'):
+              keywordList = ['extreme weather', 'floods', 'droughts']
+            
+             # show_results(keywordList)
+          with col2:
+            if st.button('Gender Keywords Search'):
+              keywordList =  ['Gender', 'Women empowernment']
+            
+             # show_results(keywordList)
+          
+          keyword = st.text_input("Please enter here \
+                                    what you want to search, \
+                                    we will look for similar context \
+                                    in the document.",
+                                    value="",)
+          if st.button("Find them."):
+            keywordList = [keyword]
+          if keywordList is not None:
+              show_results(keywordList)
+          
+
+
+
+        # @st.cache(allow_output_mutation=True)
+        # def load_sentenceTransformer(name):
+        #     return SentenceTransformer(name)
+
+        # bi_encoder = load_sentenceTransformer('msmarco-distilbert-cos-v5') # multi-qa-MiniLM-L6-cos-v1
+        # bi_encoder.max_seq_length = 64     #Truncate long passages to 256 tokens
+        # top_k = 32
+        
+        # #@st.cache(allow_output_mutation=True)
+        # #def load_crossEncoder(name):
+        #   #   return CrossEncoder(name)
+        
+        # # cross_encoder = load_crossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        # document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
+
+        # def bm25_tokenizer(text):
+        #     tokenized_doc = []
+        #     for token in text.lower().split():
+        #         token = token.strip(string.punctuation)
+
+        #         if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
+        #             tokenized_doc.append(token)
+        #     return tokenized_doc
+            
+        # def bm25TokenizeDoc(paraList):
+        #     tokenized_corpus = []
+        #     for passage in tqdm(paraList):
+        #         if len(passage.split()) >256:
+        #             temp  = " ".join(passage.split()[:256])
+        #             tokenized_corpus.append(bm25_tokenizer(temp))
+        #             temp  = " ".join(passage.split()[256:])
+        #             tokenized_corpus.append(bm25_tokenizer(temp))
+        #         else:
+        #             tokenized_corpus.append(bm25_tokenizer(passage))
+                    
+        #     return tokenized_corpus
+        
+        # tokenized_corpus = bm25TokenizeDoc(paraList)
+        
+
+        # document_bm25 = BM25Okapi(tokenized_corpus)
+        
+        # # def search(keyword):
+        # #         ##### BM25 search (lexical search) #####
+        # #         bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
+        #         top_n = np.argpartition(bm25_scores, -10)[-10:]
+        #         bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+        #         bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+                
+        #         ##### Sematic Search #####
+        #         # Encode the query using the bi-encoder and find potentially relevant passages
+        #         #query = "Does document contain {} issues ?".format(keyword)
+        #         question_embedding = bi_encoder.encode(keyword, convert_to_tensor=True)
+          
+        #         hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+        #         hits = hits[0]  # Get the hits for the first query
+                
+                
+        #         ##### Re-Ranking #####
+        #         # Now, score all retrieved passages with the cross_encoder
+        #         #cross_inp = [[query, paraList[hit['corpus_id']]] for hit in hits]
+        #         #cross_scores = cross_encoder.predict(cross_inp)
+                
+        #         # Sort results by the cross-encoder scores
+        #         #for idx in range(len(cross_scores)):
+        #           #   hits[idx]['cross-score'] = cross_scores[idx]
+                  
+                
+        #         return bm25_hits, hits
+
+        # def show_results(keywordList):
+        #   for keyword in keywordList:
+        #     bm25_hits, hits = search(keyword)     
+
+        #     st.markdown("""
+        #                 We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+        #                 """)  
+        #     # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+        #     st.markdown("Top few lexical search (BM25) hits")
+        #     for hit in bm25_hits[0:5]:
+        #         if hit['score'] > 0.00:   
+        #             st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+            
+            
+          
+          
+          
+        #   #   st.table(bm25_hits[0:3])
+            
+        #     st.markdown("\n-------------------------\n")
+        #     st.markdown("Top few Bi-Encoder Retrieval hits")
+            
+        #     hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+        #     for hit in hits[0:5]:
+        #       #  if hit['score'] > 0.45:
+        #         st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+        #     #st.table(hits[0:3]
+
+
+        # # if docs is not None:
+        # #     col1, col2 = st.columns(2)
+        # #     with col1:
+        # #       if st.button('Gender Keywords Search'):
+        # #         keywordList =  ['Gender Equality', 'Women empowernment']
+        # #         show_results(keywordList)
+        # #     with col2:
+        # #       if st.button('Climate Change Keyword Search'):
+        # #         keywordList = ['extreme weather', 'floods', 'droughts']
+        # #         show_results(keywordList)
+            
+        # #     keyword = st.text_input("Please enter here \
+        # #                              what you want to search, \
+        # #                              we will look for similar context \
+        # #                              in the document.",
+        # #                              value="",)
+        # #     if st.button("Find them."):
+        # #       show_results([keyword])
+
+  
+            # choice1 = st.radio(label = 'Keyword Search',
+            #               help = 'Search  \
+            #               or else you can try a example document', 
+            #               options = ('Enter your own Query', 'Try Example'), 
+            #               horizontal = True)
+            
+            # if choice1 == 'Enter your own Query':
+            #   keyword = st.text_input("Please enter here \
+            #                         what you want to search, \
+            #                         we will look for similar context \
+            #                         in the document.",
+            #                         value="",)
+            # else:
+            #   option1 = st.selectbox('Select the Predefined word cluster',
+            #                     ('Gender:[Gender Equality, Women empowernment]', 
+            #                     'Climate change:[extreme weather, floods, droughts]',
+            #                     ))
+            #   if option1 == 'Gender:[Gender Equality, Women empowernment]':
+            #     keywordList = ['Gender Equality', 'Women empowernment']
+            #   else:
+            #     keywordList = ['extreme weather', 'floods', 'droughts']
+
+            # option1 = st.selectbox('Select the Predefined word cluster',
+            #                     ('Gender:[Gender Equality, Women empowernment]', 
+            #                     'Climate change:[extreme weather, floods, droughts]',
+            # #                     'Enter your Own Keyword Query'))
+            # if option1 == 'Enter your Own Keyword Query':
+            #   keyword = st.text_input("Please enter here \
+            #                         what you want to search, \
+            #                         we will look for similar context \
+            #                         in the document.",
+            #                         value="",)
+            # elif option1 == 'Gender:[Gender Equality, Women empowernment]':
+            #   keywordList = ['Gender Equality', 'Women empowernment']
+            # elif option1 == 'Climate change:[extreme weather, floods, droughts]':
+            #   keywordList = ['extreme weather', 'floods', 'droughts']
+
+
+            # st.markdown("### 📌 Step Two: Search Keyword in Document ### ")             
+            
+                                      
+            # @st.cache(allow_output_mutation=True)
+            # def load_sentenceTransformer(name):
+            #     return SentenceTransformer(name)
+
+            # bi_encoder = load_sentenceTransformer('msmarco-distilbert-cos-v5') # multi-qa-MiniLM-L6-cos-v1
+            # bi_encoder.max_seq_length = 64     #Truncate long passages to 256 tokens
+            # top_k = 32
+            
+            # #@st.cache(allow_output_mutation=True)
+            # #def load_crossEncoder(name):
+            #   #   return CrossEncoder(name)
+            
+            # # cross_encoder = load_crossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+            # document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
+
+            # def bm25_tokenizer(text):
+            #     tokenized_doc = []
+            #     for token in text.lower().split():
+            #         token = token.strip(string.punctuation)
+
+            #         if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
+            #             tokenized_doc.append(token)
+            #     return tokenized_doc
+                
+            # def bm25TokenizeDoc(paraList):
+            #     tokenized_corpus = []
+            #     for passage in tqdm(paraList):
+            #         if len(passage.split()) >256:
+            #             temp  = " ".join(passage.split()[:256])
+            #             tokenized_corpus.append(bm25_tokenizer(temp))
+            #             temp  = " ".join(passage.split()[256:])
+            #             tokenized_corpus.append(bm25_tokenizer(temp))
+            #         else:
+            #             tokenized_corpus.append(bm25_tokenizer(passage))
+                        
+            #     return tokenized_corpus
+            
+            # tokenized_corpus = bm25TokenizeDoc(paraList)
+            
+
+            # document_bm25 = BM25Okapi(tokenized_corpus)
+            
+            
+            # def search(keyword):
+            #     ##### BM25 search (lexical search) #####
+            #     bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
+            #     top_n = np.argpartition(bm25_scores, -10)[-10:]
+            #     bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+            #     bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+                
+            #     ##### Sematic Search #####
+            #     # Encode the query using the bi-encoder and find potentially relevant passages
+            #     #query = "Does document contain {} issues ?".format(keyword)
+            #     question_embedding = bi_encoder.encode(keyword, convert_to_tensor=True)
+          
+            #     hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+            #     hits = hits[0]  # Get the hits for the first query
+                
+                
+            #     ##### Re-Ranking #####
+            #     # Now, score all retrieved passages with the cross_encoder
+            #     #cross_inp = [[query, paraList[hit['corpus_id']]] for hit in hits]
+            #     #cross_scores = cross_encoder.predict(cross_inp)
+                
+            #     # Sort results by the cross-encoder scores
+            #     #for idx in range(len(cross_scores)):
+            #       #   hits[idx]['cross-score'] = cross_scores[idx]
+                  
+                
+            #     return bm25_hits, hits
+
+            # def show_results(keywordList):
+            #   for keyword in keywordList:
+            #     bm25_hits, hits = search(keyword)     
+
+            #     st.markdown("""
+            #                 We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+            #                 """)  
+            #     # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+            #     st.markdown("Top few lexical search (BM25) hits")
+            #     for hit in bm25_hits[0:5]:
+            #         if hit['score'] > 0.00:   
+            #             st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                
+                
+              
+              
+              
+            #   #   st.table(bm25_hits[0:3])
+                
+            #     st.markdown("\n-------------------------\n")
+            #     st.markdown("Top few Bi-Encoder Retrieval hits")
+                
+            #     hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+            #     for hit in hits[0:5]:
+            #       #  if hit['score'] > 0.45:
+            #         st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+            #     #st.table(hits[0:3]
+        
+
+          
+
+            # # if st.button("Find them."):
+            # #     bm25_hits, hits = search(keyword)     
+
+            # #     st.markdown("""
+            # #                 We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+            # #                 """)  
+            # #     # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+            # #     st.markdown("Top few lexical search (BM25) hits")
+            # #     for hit in bm25_hits[0:5]:
+            # #         if hit['score'] > 0.00:   
+            # #             st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                
+                
+              
+              
+              
+            # #   #   st.table(bm25_hits[0:3])
+                
+            # #     st.markdown("\n-------------------------\n")
+            # #     st.markdown("Top few Bi-Encoder Retrieval hits")
+                
+            # #     hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+            # #     for hit in hits[0:5]:
+            # #       #  if hit['score'] > 0.45:
+            # #         st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+            # #     #st.table(hits[0:3]
+        
+
+          

appStore/multiapp.py

@@ -0,0 +1,51 @@
+"""Frameworks for running multiple Streamlit applications as a single app.
+"""
+import streamlit as st
+from PIL import Image
+
+class MultiApp:
+    """Framework for combining multiple streamlit applications.
+    Usage:
+        def foo():
+            st.title("Hello Foo")
+        def bar():
+            st.title("Hello Bar")
+        app = MultiApp()
+        app.add_app("Foo", foo)
+        app.add_app("Bar", bar)
+        app.run()
+    It is also possible keep each application in a separate file.
+        import foo
+        import bar
+        app = MultiApp()
+        app.add_app("Foo", foo.app)
+        app.add_app("Bar", bar.app)
+        app.run()
+    """
+    def __init__(self):
+        self.apps = []
+
+    def add_app(self, title, func):
+        """Adds a new application.
+        Parameters
+        ----------
+        func:
+            the python function to render this app.
+        title:
+            title of the app. Appears in the dropdown in the sidebar.
+        """
+        self.apps.append({
+            "title": title,
+            "function": func
+        })
+
+    def run(self):
+        st.sidebar.write(format_func=lambda app: app['title'])
+        image = Image.open('appStore/img/giz_sdsn.jpg')
+        st.sidebar.image(image)
+        app = st.sidebar.radio(
+            'Go To',
+            self.apps,
+            format_func=lambda app: app['title'])
+
+        app['function']()

appStore/sdg_analysis.py

@@ -0,0 +1,204 @@
+# set path
+import glob, os, sys; sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+
+import tempfile
+import sqlite3
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> SDSN x GIZ Policy Action Tracking v0.1</h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=True):
+
+        st.write(
+            """     
+            The *Analyse Policy Document* app is an easy-to-use interface built in Streamlit for analyzing policy documents - developed by GIZ Data and the Sustainable Development Solution Network. \n
+                1. Keyword heatmap \n
+                2. SDG Classification for the paragraphs/texts in the document
+            """
+        )
+
+        st.markdown("")
+
+    st.markdown("")
+    st.markdown("##  📌 Step One: Upload document ")
+    
+    with st.container():
+
+
+      docs = None
+        # asking user for either upload or select existing doc
+      choice = st.radio(label = 'Select the Document',
+                        help = 'You can upload the document \
+                        or else you can try a example document', 
+                        options = ('Upload Document', 'Try Example'), 
+                        horizontal = True)
+
+      if choice == 'Upload Document':
+        uploaded_file = st.file_uploader('Upload the File', type=['pdf', 'docx', 'txt'])
+        if uploaded_file is not None:
+          with tempfile.NamedTemporaryFile(mode="wb") as temp:
+              bytes_data = uploaded_file.getvalue()
+              temp.write(bytes_data)
+
+              st.write("Uploaded Filename: ", uploaded_file.name)
+              file_name =  uploaded_file.name
+              file_path = temp.name
+              docs = pre.load_document(file_path, file_name)
+              docs_processed, df, all_text, par_list = clean.preprocessingForSDG(docs)
+              #haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+      else:
+        # listing the options
+        option = st.selectbox('Select the example document',
+                              ('Ethiopia: 10 Year Development Plan',
+                              'South Africa:Low Emission strategy'))
+        if option is 'South Africa:Low Emission strategy':
+          file_name = file_path  = 'sample/South Africa_s Low Emission Development Strategy.txt'
+          st.write("Selected document:", file_name.split('/')[1])
+          # with open('sample/South Africa_s Low Emission Development Strategy.txt') as dfile:
+          # file = open('sample/South Africa_s Low Emission Development Strategy.txt', 'wb')
+        else:
+          # with open('sample/Ethiopia_s_2021_10 Year Development Plan.txt') as dfile:
+          file_name = file_path =  'sample/Ethiopia_s_2021_10 Year Development Plan.txt'
+          st.write("Selected document:", file_name.split('/')[1])
+        
+        if option is not None:
+          docs = pre.load_document(file_path,file_name)
+          # haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+          docs_processed, df, all_text, par_list = clean.preprocessingForSDG(docs)
+
+        
+        
+      if docs is not None:
+
+                @st.cache(allow_output_mutation=True)
+                def load_keyBert():
+                    return KeyBERT()
+
+                kw_model = load_keyBert()
+
+                keywords = kw_model.extract_keywords(
+                all_text,
+                keyphrase_ngram_range=(1, 2),
+                use_mmr=True,
+                stop_words="english",
+                top_n=15,
+                diversity=0.7,
+                )
+
+                st.markdown("## 🎈 What is my document about?")
+            
+                df = (
+                    DataFrame(keywords, columns=["Keyword/Keyphrase", "Relevancy"])
+                    .sort_values(by="Relevancy", ascending=False)
+                    .reset_index(drop=True)
+                )
+
+                df.index += 1
+
+                # Add styling
+                cmGreen = sns.light_palette("green", as_cmap=True)
+                cmRed = sns.light_palette("red", as_cmap=True)
+                df = df.style.background_gradient(
+                    cmap=cmGreen,
+                    subset=[
+                        "Relevancy",
+                    ],
+                )
+                c1, c2, c3 = st.columns([1, 3, 1])
+
+                format_dictionary = {
+                    "Relevancy": "{:.1%}",
+                }
+
+                df = df.format(format_dictionary)
+
+                with c2:
+                    st.table(df) 
+
+                ######## SDG classiciation
+                # @st.cache(allow_output_mutation=True)
+                # def load_sdgClassifier():
+                #     classifier = pipeline("text-classification", model= "../models/osdg_sdg/")
+
+                #     return classifier
+                
+                # load from disc (github repo) for performance boost
+                @st.cache(allow_output_mutation=True)
+                def load_sdgClassifier():
+                    classifier = pipeline("text-classification", model= "jonas/sdg_classifier_osdg")
+
+                    return classifier
+
+                classifier = load_sdgClassifier()
+
+                # # not needed, par list comes from pre_processing function already
+
+                # word_list = all_text.split()
+                # len_word_list = len(word_list)
+                # par_list = []
+                # par_len = 130
+                # for i in range(0,len_word_list // par_len):
+                #     string_part = ' '.join(word_list[i*par_len:(i+1)*par_len])
+                #     par_list.append(string_part)
+                    
+                labels = classifier(par_list)
+                labels_= [(l['label'],l['score']) for l in labels]
+                df = DataFrame(labels_, columns=["SDG", "Relevancy"])
+                df['text'] = par_list      
+                df = df.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)  
+                df.index += 1
+                df =df[df['Relevancy']>.85]
+                x = df['SDG'].value_counts()
+
+                plt.rcParams['font.size'] = 25
+                colors = plt.get_cmap('Blues')(np.linspace(0.2, 0.7, len(x)))
+                # plot
+                fig, ax = plt.subplots()
+                ax.pie(x, colors=colors, radius=2, center=(4, 4),
+                    wedgeprops={"linewidth": 1, "edgecolor": "white"}, frame=False,labels =list(x.index))
+
+                st.markdown("## 🎈 Anything related to SDGs?")
+
+                c4, c5, c6 = st.columns([5, 7, 1])
+
+                # Add styling
+                cmGreen = sns.light_palette("green", as_cmap=True)
+                cmRed = sns.light_palette("red", as_cmap=True)
+                df = df.style.background_gradient(
+                    cmap=cmGreen,
+                    subset=[
+                        "Relevancy",
+                    ],
+                )
+
+                format_dictionary = {
+                    "Relevancy": "{:.1%}",
+                }
+
+                df = df.format(format_dictionary)
+
+                with c4:
+                    st.pyplot(fig)
+                with c5:
+                    st.table(df) 
+                
+                

ndcs/cca.txt

@@ -0,0 +1,81 @@
+{"climate_risks_droughts": {"category": "climate change adaptation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+                                                                            1: "Droughts are not climate risks concerns",
+								            2:  "Droughts are among the five climate risks concerns"}},
+"climate_risks_extreme_weather": {"category": "climate change adaptation", "id": {0:"(I)NDC not submitted or not yet included in analysis",
+                                                                            1: "Extreme Weathers are not climate risks concerns",
+								            2:  "Extreme Weathers are among the five climate risks concerns"}},
+"climate_risks_floods": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Floods are not climate risks concerns",
+								         2:  "Floods are among the five climate risks concerns"}},
+"climate_risks_temp_increase": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Temperature increase are not climate risks concerns",
+								         2:  "Temperature increase are among the five climate risks concerns"}},
+"climate_risks_sea_level_rise": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Sea level rise is not a climate risks concerns",
+								         2:  "Sea level rise is among the five climate risks concerns"}},
+
+"priority_sectors_agriculture": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+										1: "Agricultural sector is not that important in the context of adaptation ambitions",
+										2: "In the context of adaptation ambitions Agricultural sector is very important for the country",
+										3: "Agriculture sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"priority_sectors_ecosystems": {"category": "climate change adaptation","id": {0 :"(I)NDC not submitted or not yet included in analysis",
+ 									       1 :"Biodiversity and preservation of Ecosystems is not that important in the context of adaptation ambitions",
+									       2: "In the context of adaptation ambitions Biodiversity and preservation of Ecosystems is very important for the country",
+									       3: "Biodiversity and Ecosystems plays an importance for the country, and therefore in the adaptation ambitions Biodiversity and Ecosystems has special actions and aims"}},
+"priority_sectors_forestry": {"category": "climate change adaptation", "id": {0: "(I)NDC not submitted or not yet included in analysis",
+									      1: "Forestry sector is not that important in the context of adaptation ambitions",
+									      2: "In the context of adaptation ambitions Forestry sector is very important for the country",
+									      3: "Forestry sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+"priority_sectors_health": {"category": "climate change adaptation","id": { 0: "(I)NDC not submitted or not yet included in analysis",
+									    1: "Health sector is not that important in the context of adaptation ambitions",
+									    2: "In the context of adaptation ambitions Health sector is very important for the country",
+									    3: "Health sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"priority_sectors_water": {"category": "climate change adaptation","id": { 0 : "(I)NDC not submitted or not yet included in analysis",
+ 									   1: "Water sector is not that important in the context of adaptation ambitions",
+									   2: "In the context of adaptation ambitions Water sector is very important for the country",
+									   3: "Water sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"vulnerability_agriculture": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									     1: "Agriculture is a not a vulnerable sector",
+   									     2: "Agriculture is a vulnerable sector"}},
+"vulnerability_coastal_zones": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									       1: "Coastal Zone is a not a vulnerable sector",
+   									       2: "Coastal Zone is a vulnerable sector"}},
+"vulnerability_ecosystems": {"category": "climate change adaptation", "id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									     1: "Biodiversity and Ecosystems is a not a vulnerable sector",
+   									     2: "Biodiversity and Ecosystems is a vulnerable sector"}},
+ "vulnerability_health": {"category": "climate change adaptation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+									 1: "Health is a not a vulnerable sector",
+   									 2: "Health is a vulnerable sector"}},
+ "vulnerability_water": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+    									1: "Water is a not a vulnerable sector",
+   									2: "Water is a vulnerable sector"}},
+
+"costs_of_adaptation": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+								       1: "The partial cost of adaptation is tentatively around few million dollars",
+								       2: " The cost of adaptation will be 0-1 billion US$ until 2030",
+								       3: " The cost of adaptation will be 1-5 billion US$ until 2030",
+								       4: " The cost of adaptation will be 5-10 billion US$ until 2030",
+								       5: " The cost of adaptation will be 10-20 billion US$ until 2030",
+								       6:  "The cost of adaptation will be more than 20 billion US$ until 2030"}},
+"costs_of_future_climate_related_hazards": {"category": "climate change adaptation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+											   1: "The future losses from climate change will be huge",
+   											   2: "The climate hazards cause significant loss to economy and life, and the cost of Future losses could go around few million dollars"}},
+
+"costs_of_recent_climate_related_hazards": {"category": "climate change adaptation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+											   1: "No losses indicated",
+   									                   2: "In the recent climate hazards there has been significant Economic losses.",
+  										           3: "In the recent climate hazards the impact on human life has been significant",
+											   4: "In the recent climate hazards the impact on human life has been significant and the economic loss amounts to 5.3"}},
+"quantified_adaptation_targets": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+										 1:"No quantitative adaptation target",
+										 2: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years",
+										 3: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years",
+										 4: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years"}},
+
+"slow_onset_others": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+							             1:"Apart from sea level rise and temperature increase, no other specific slow onset process",
+							             2: "There are other slow onset processes additional to sea level rise and temperature increase like loss of biodiversity, desertification, glacier retreat, salinisation or ocean acidification"}},
+}

ndcs/ccm.txt

@@ -0,0 +1,86 @@
+{"agriculture": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+							        1: "Agriculture sector is not considered for climate change mitigation",
+								2: "Agriculture sector contribution in greenhouse gases emission is significant and therefore is part of climate change mitigation",
+								3: "Agriculture sector contribution in greenhouse gases emission is significant. Given the importance of agriculture sector for economy and and its adverse contribution in greenhouse gas emissions it is a Focus area for climate change mitigation and needs to be prioritised"}},
+
+"energy_efficiency": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+							        1: "Energy Efficiency is not considered for climate change mitigation",
+								2: "Energy sector contribution in greenhouse gases emission is significant and therefore Energy Efficiency is part of climate change mitigation",
+								3: "Energy sector contribution in greenhouse gases emission is significant. Given the importance of the energy sector for economy and its adverse contribution to greenhouse gas emissions, energy efficiency is a Focus area for climate change mitigation and needs to be prioritised. The quantified renewable energy targets like for example in solar, geothermal, wind power are provided."}},
+
+"fossil_fuel_production": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+									  1:"There is no recorded FFP (2016)",
+									  2: "Fossil fuel Production is important for economy",
+									  3:"Fossil fuel Production is important to provide for the basic requirements of the people in the country",
+									  4:"The country's dependence on Fossil fuel production to meet energy and other requirements cannot be ignored, however the climate change impact due to the same cannot be ignored. The plans and actions as part of climate change mitigation includes measures to address production (e.g. subsidy removal; taxes); cleaner production",
+								          5: "Fossil fuel Production is important to provide for the basic requirements of the people in the country.The country's dependence on Fossil fuel production to meet energy and other requirements cannot be ignored, however the climate change impact due to same cannot be ignored. The plans and actions as part of climate change mitigation includes measures to address production (e.g. subsidy removal; taxes); cleaner production"}},
+"fossil_fuel_subsidiaries": {"category": "climate change mitigation","id":{0: "(I)NDC not submitted or not yet included in analysis",
+									   1:"fossil Fuel subsidiaries are not considered",
+									   2:"the alternates/subsidiaries to fossil Fuel need to be considered to meet the mitigations ambitions",
+									   3:"The fossil fuel contribution towards greenhouse gas emissions is very high and therefore there is a need to find the alternatives/substitutes for the same. The replacement of fossil fuels with alternates is a priority focus area in the mitigation actions to meet mitigation ambitions."}},
+
+"land_use_and_forestry": {"category": "climate change mitigation", "id":{0:"(I)NDC not submitted or not yet included in analysis",
+  									 1:"land use and forestry are not considered",
+									 2:"the land use and forestry contribute to greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+									 3:"The land use and forestry contribution towards greenhouse gas emissions is significant and therefore there is need to quantify the mitigation potential land use and forestry."}},
+"land_use_change": {"category": "climate change mitigation", "id": {0:"(I)NDC not submitted or not yet included in analysis",
+								    1: "land use change Not mentioned",
+							            2: "land use change is being considered, but there are no mitigation targets",
+								    3: "land use change is being considered as part of mitigation targets",
+								    4: "land use change can play an important role in mitigation efforts. As part of mitigation plan there are quantified targets for land use change.",
+								    5: "land use change can play an important role in mitigation efforts. As part of mitigation plan there are quantified targets for land use change."}},
+
+"renewable_energy": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+								     1:"renewable energy is not considered",
+							             2:"Renewable energy are direct measure to reduce the greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+								     3:"Renewable energy are direct measure to reduce the greenhouse gas emissions and therefore there is need to quantify the mitigation potential in terms of renewable energy targets and specific sub-sectors of action (e.g. solar, geothermal, wind power)"}},
+
+"temp_target": {"category": "climate change mitigation", "id": { 0: "(I)NDC not submitted or not yet included in analysis",
+ 								 1:"Not mentioning global effort to limit global temperature increase to 2 degree celsius or 1.5 degree C",
+								 2:"there is urgent need to limit global temperature increase to 2 degree celsius",						
+								 3:"there is urgent need to limit global temperature increase to 1.5 degree C",
+							         4:"there is urgent need to limit global temperature increase to 2 degree celsius",
+ 								 5:"there is urgent need to limit global temperature increase to 1.5 degree C"}},
+"waste": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							 1:"Waste as a topic is not mentioned",
+ 							 2:"Waste reduction or management can play important role in mitigation plan and ambitions",
+							 3:"Waste reduction or management can play an important role in sustainable development and hence is a focus area in mitigation plan and ambitions"}},
+"transport": {"category": "climate change mitigation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+							     1:"Transport is not considered",
+							     2:"Transport contribute to greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+							     3:"transport sector contribution towards greenhouse gas emissions is significant and therefore there is need to focus/prioritise the transport sector to meet the mitigation potential"}},
+
+"reducing_non_co2_gases": {"category": "climate change mitigation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+									  1:"Reduction of non CO2 gases not indicated",
+									  2:"Efforts should be made in reduction of NOn CO2 gases too."}},
+
+
+"base_year": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							     1: "No base year",
+     							     2: "the base year or reference point for measurement of emissions is year 19XX"}},
+
+"carbon_capture_and_storage": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+									      1: "carbon capture and storage not indicated",
+									      2:"With Technology advancement the mitigation efforts can also in form of carbon capture and storage.",
+									      3: "With technological advancement the mitigation efforts can also be in form of carbon capture and storage. This should be  a focus area and more options need to be explored to do carbon capture and storage."}},
+
+"costs_of_ccm": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+								1: "(partial) costs not indicated",
+								2: " the mitigation actions and efforts will cost 0-1 billion US$ until 2030",
+								3:"the mitigation actions and efforts will cost 1-5 billion US$ until 2030",
+								4:"the mitigation actions and efforts will cost5-10 billion US$ until 2030",
+								5: "the mitigation actions and efforts will cost 10-20 billion US$ until 2030",
+								6:"the mitigation actions and efforts will cost will be more than 20 billion US$ until 2030"}},
+
+"market_mechanisms": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+								     1: "International market mechanisms not mentioned",
+ 								     2:"One good mechanism to deal with greenhouse gas emissions is to explore International market mechanisms",
+								     3: "International market mechanisms are not a good way of dealing with mitigation ambitions and therefore should not be considered. Greenhouse gas emissions cannot be part of tradable commodity.",
+								     4: "Carbon emissions of greenhouse gas emissions are now a tradable commodity and these can provide a good source for funds and achieving mitigation ambitions. Therefore it is important to exploreInternational market mechanisms. It is important that such means should be explored and there will be plan of actions soon to include these in meeting mitigations target",
+								     5: "Carbon emissions of greenhouse gas emissions are now a tradable commodity and these can provide a good source for funds and achieving mitigation ambitions. Therefore it is important to exploreInternational market mechanisms. It is important that such means should be explored and there will be plan of actions soon to include these in meeting mitigations target"}},
+
+"redd": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							1: "REDD+ not mentioned",
+							2: "Reducing Emissions of Deforestation and Forest Degradation/REDD+",
+							3: "Reducing Emissions of Deforestation and Forest Degradation/REDD+"}},
+}

ndcs/countryList.txt

@@ -0,0 +1,170 @@
+{'Afghanistan': 'AFG',
+ 'Albania': 'ALB',
+ 'Algeria': 'DZA',
+ 'Andorra': 'AND',
+ 'Angola': 'AGO',
+ 'Antigua and Barbuda': 'ATG',
+ 'Argentina': 'ARG',
+ 'Armenia': 'ARM',
+ 'Australia': 'AUS',
+ 'Azerbaijan': 'AZE',
+ 'Bahamas': 'BHS',
+ 'Bahrain': 'BHR',
+ 'Bangladesh': 'BGD',
+ 'Barbados': 'BRB',
+ 'Belarus': 'BLR',
+ 'Belize': 'BLZ',
+ 'Benin': 'BEN',
+ 'Bhutan': 'BTN',
+ 'Bolivia': 'BOL',
+ 'Bosnia and Herzegovina': 'BIH',
+ 'Botswana': 'BWA',
+ 'Brazil ': 'BRA',
+ 'Brunei Darussalam': 'BRN',
+ 'Burkina Faso': 'BFA',
+ 'Burundi ': 'BDI',
+ 'Cabo Verde': 'CPV',
+ 'Cambodia': 'KHM',
+ 'Cameroon': 'CMR',
+ 'Canada': 'CAN',
+ 'Central African Republic': 'CAF',
+ 'Chad': 'TCD',
+ 'Chile': 'CHL',
+ 'China': 'CHN',
+ 'Colombia': 'COL',
+ 'Comoros': 'COM',
+ 'Congo': 'COG',
+ 'Cook Islands': 'COK',
+ 'Costa Rica': 'CRI',
+ 'Cote dIvoire': 'CIV',
+ 'Cuba': 'CUB',
+ "Democratic People's Republic of Korea": 'PRK',
+ 'Democratic Republic of Congo': 'COD',
+ 'Djibouti': 'DJI',
+ 'Dominica': 'DMA',
+ 'Dominican Republic': 'DOM',
+ 'Ecuador': 'ECU',
+ 'Egypt': 'EGY',
+ 'El Salvador': 'SLV',
+ 'Equatorial Guinea': 'GNQ',
+ 'Eritrea': 'ERI',
+ 'Ethiopia': 'ETH',
+ 'European Union': 'EU',
+ 'Fiji': 'FJI',
+ 'Gabon': 'GAB',
+ 'Gambia': 'GMB',
+ 'Georgia': 'GEO',
+ 'Ghana': 'GHA',
+ 'Grenada': 'GRD',
+ 'Guatemala': 'GTM',
+ 'Guinea': 'GIN',
+ 'Guinea Bissau': 'GNB',
+ 'Guyana': 'GUY',
+ 'Haiti': 'HTI',
+ 'Honduras': 'HND',
+ 'Iceland': 'ISL',
+ 'India': 'IND',
+ 'Indonesia': 'IDN',
+ 'Iran': 'IRN',
+ 'Iraq': 'IRQ',
+ 'Israel': 'ISR',
+ 'Jamaica': 'JAM',
+ 'Japan': 'JPN',
+ 'Jordan': 'JOR',
+ 'Kazakhstan': 'KAZ',
+ 'Kenya': 'KEN',
+ 'Kingdom of Eswatini': 'SWZ',
+ 'Kiribati': 'KIR',
+ 'Kuwait': 'KWT',
+ 'Kyrgyzstan': 'KGZ',
+ 'Lao Peoples Democratic Republic': 'LAO',
+ 'Lebanon': 'LBN',
+ 'Lesotho': 'LSO',
+ 'Liberia': 'LBR',
+ 'Libya': 'LBY',
+ 'Liechtenstein': 'LIE',
+ 'Madagascar': 'MDG',
+ 'Malawi': 'MWI',
+ 'Malaysia': 'MYS',
+ 'Maldives': 'MDV',
+ 'Mali': 'MLI',
+ 'Marshall Islands': 'MHL',
+ 'Mauritania': 'MRT',
+ 'Mauritius': 'MUS',
+ 'Mexico': 'MEX',
+ 'Micronesia': 'FSM',
+ 'Monaco': 'MCO',
+ 'Mongolia': 'MNG',
+ 'Montenegro': 'MNE',
+ 'Morocco': 'MAR',
+ 'Mozambique': 'MOZ',
+ 'Myanmar': 'MMR',
+ 'Namibia': 'NAM',
+ 'Nauru': 'NRU',
+ 'Nepal': 'NPL',
+ 'New Zealand': 'NZL',
+ 'Nicaragua': 'NIC',
+ 'Niger': 'NER',
+ 'Nigeria': 'NGA',
+ 'Niue': 'NIU',
+ 'Norway': 'NOR',
+ 'Oman': 'OMN',
+ 'Pakistan': 'PAK',
+ 'Palau ': 'PLW',
+ 'Palestine': 'PSE',
+ 'Panama': 'PAN',
+ 'Papua New Guinea': 'PNG',
+ 'Paraguay': 'PRY',
+ 'Peru': 'PER',
+ 'Philippines': 'PHL',
+ 'Qatar': 'QAT',
+ 'Republic of Moldova': 'MDA',
+ 'Republic of North Macedonia': 'MKD',
+ 'Russian Federation': 'RUS',
+ 'Rwanda': 'RWA',
+ 'Saint Kitts and Nevis': 'KNA',
+ 'Saint Lucia': 'LCA',
+ 'Saint Vincent and the Grenadines': 'VCT',
+ 'Samoa': 'WSM',
+ 'San Marino': 'SMR',
+ 'Sao Tome and Principe': 'STP',
+ 'Saudi Arabia': 'SAU',
+ 'Senegal': 'SEN',
+ 'Serbia': 'SRB',
+ 'Seychelles': 'SYC',
+ 'Sierra Leone': 'SLE',
+ 'Singapore': 'SGP',
+ 'Solomon Islands': 'SLB',
+ 'Somalia': 'SOM',
+ 'South Africa': 'ZAF',
+ 'South Korea': 'KOR',
+ 'South Sudan': 'SSD',
+ 'Sri Lanka': 'LKA',
+ 'Sudan': 'SDN',
+ 'Suriname': 'SUR',
+ 'Switzerland': 'CHE',
+ 'Syria': 'SYR',
+ 'Tajikistan': 'TJK',
+ 'Thailand': 'THA',
+ 'Timor Leste': 'TLS',
+ 'Togo': 'TGO',
+ 'Tonga': 'TON',
+ 'Trinidad and Tobago': 'TTO',
+ 'Tunisia': 'TUN',
+ 'Turkey': 'TUR',
+ 'Turkmenistan': 'TKM',
+ 'Tuvalu': 'TUV',
+ 'Uganda': 'UGA',
+ 'Ukraine': 'UKR',
+ 'United Arab Emirates': 'ARE',
+ 'United Kingdom': 'GBR',
+ 'United Republic of Tanzania': 'TZA',
+ 'United States of America': 'USA',
+ 'Uruguay': 'URY',
+ 'Uzbekistan': 'UZB',
+ 'Vanuatu': 'VUT',
+ 'Venezuela ': 'VEN',
+ 'Vietnam': 'VNM',
+ 'Yemen': 'YEM',
+ 'Zambia': 'ZMB',
+ 'Zimbabwe': 'ZWE'}

packages.txt

@@ -0,0 +1,2 @@
+poppler-utils
+xpdf

requirements.txt

@@ -0,0 +1,14 @@
+farm-haystack
+farm-haystack[ocr]
+spacy==3.2.0
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0.tar.gz#egg=en_core_web_sm
+keybert==0.5.1
+matplotlib==3.5.1
+nltk==3.7
+numpy==1.22.1
+pandas==1.4.0
+pdfplumber==0.6.2
+Pillow==9.1.1
+seaborn==0.11.2
+transformers==4.13.0
+rank_bm25

sample/Ethiopia_s_2021_10 Year Development Plan.txt

@@ -0,0 +1,737 @@
+Ethiopia 2030: The Pathway to Prosperity
+Ten Years Perspective Development Plan (2021 � 2030)
+1. Baselines and Assumptions
+2. Strategic pillars
+3. Departures
+4. Macroeconomic goals
+5. Implications of the COVID-19 pandemic and necessary mitigation measures
+6. Potentials/capabilities
+7. Focus areas
+7.1. Productive sectors
+7.2. Services sector
+7.3. Enabling sectors
+8. Balanced and competitive development (nationally, regionally and locally)
+9. Monitoring and Evaluation
+Content
+1. Baselines and Assumptions
+Poverty Reduction (%)
+Key performances of previous years
+45.5 44.2
+38.7
+29.6
+23.5
+19
+0
+5
+10
+15
+20
+25
+30
+35
+40
+45
+50
+1994 2000 2005 2011 2016 2020
+Percent
+Year
+Proportion of people living below poverty line
+10.5
+8.8
+10.1
+7.7
+9
+5.19-6.20
+0 2 4 6 8 10 12
+GTP I: 2011-2015
+GTP II: 2015/16
+GTP II: 2016/17
+GTP II: 2017/18
+GTP II: 2018/19
+GTP II: 2019/20 (projection, with
+COVID-19)
+GDP growth rate (%)
+1. Baselines and Assumptions
+Share of economic sectors in GDP (%) Merchandise export as % of GDP
+8.66
+7.33
+6.57
+5.93
+4.91
+3.86 3.56 3.37
+2.77
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+Percent
+Year
+46.9
+45
+43.5
+41.4
+39.5
+37.1 35.9
+34.5
+32.8
+13.4
+15
+17.3
+18.8
+21
+23.5
+25.7 26.9 27.8
+4.7 4.8 5 5.3 5.6 6.1 6.9 6.8 6.8
+7.1
+8.6
+10.7 12
+14.2
+16.2
+17.8 19.1 20.1
+39.8 40.1 39.2 39.8 39.4 38.4 38.6 39.4
+0
+5
+10
+15
+20
+25
+30
+35
+40
+45
+50
+2010/11 2011/12 2012/13 2013/14 2014/15 2015/16 2016/17 2017/18 2018/19
+Percent
+Agriculture Industry Manufacturing Construction Services
+1. Baselines and Assumptions
+Labour force participation (2013)
+73%
+7%
+20%
+Agriculture
+Industry
+Services
+7%
+22%
+71%
+Agriculture
+Industry
+Services
+Urban labour force participation (2013)
+1. Baselines and Assumptions
+High and increasing Unemployment Rate
+� Urban unemployment rate = 19.1% in 2018
+� Youth unemployment rate = 25.3 %
+? Male = 18.6%
+? Female 30.9 %
+� Rural unemployment rate = 2% in 2013
+� Declining per capita rural land creating
+disguised unemployment
+402,869
+471,535
+Male Female Total Male Female Total
+2014 2018
+15-19 yr. 20-24 yr. 25-29 yr. Linear (20-24 yr.)
+Number of unemployed people in urban areas
+1. Baselines and Assumptions
+Challenges
+1. Macroeconomic imbalances
+?Sustained high inflation
+?High and rising unemployment especially
+in urban areas
+?High and rising debt burden
+?Chronic foreign currency shortage
+?Sluggish (though encouraging) rate of
+structural change
+2. Vulnerability to shocks (COVID-19, Climate
+changes, Desert Locust infestation, etc)
+3. Poor quality and high inequity in
+infrastructure projects
+4. Poor quality services in health and
+education
+� High repetition and dropout rates from school
+1. Baselines and Assumptions
+� Poor quality of growth and slow
+structural change
+� Excessive aid and loan
+dependence for financing
+infrastructural and construction
+investments
+� Limited success in expanding
+manufacturing and modern
+agriculture which have high job
+creation potentials
+� Weak institutional capacity as
+the main culprit of all failures
+? Provision of quality services
+(electricity, water, telephone,
+internet)
+? Creation of enough jobs and
+improved living standards
+? Generation of reliable foreign
+exchange revenue and debtsustainable
+national economic
+capacity
+? Completion of development
+projects and investment plans
+under public-private
+partnerships
+� Low reward for merit, productivity and effort
+while low disincentive for laziness, wastefulness
+and corruption
+� Slow institutional change and transformation in:
+? Government policies
+? Investor attitude
+? Youth behaviour
+? Role of the intellectuals
+� The need for sustained increase in production
+and productivity
+� The need to set a common national vision to
+achieve major successes with consensus and
+popular legitimacy
+Major areas of failure in the economy
+1. Baselines and Assumptions
+� Poor quality of growth and slow
+structural change
+� Excessive aid and loan
+dependence for financing
+infrastructural and construction
+investments
+� Limited success in expanding
+manufacturing and modern
+agriculture which have high job
+creation potentials
+� Weak institutional capacity as
+the main culprit of all failures
+? Provision of quality services
+(electricity, water, telephone,
+internet)
+? Creation of enough jobs and
+improved living standards
+? Generation of reliable foreign
+exchange revenue and debtsustainable
+national economic
+capacity
+? Completion of development
+projects and investment plans
+under public-private
+partnerships
+� Low reward for merit, productivity and effort
+while low disincentive for laziness, wastefulness
+and corruption
+� Slow institutional change and transformation in:
+? Government policies
+? Investor attitude
+? Youth behaviour
+? Role of the intellectuals
+� The need for sustained increase in production
+and productivity
+� The need to set a common national vision to
+achieve major successes with consensus and
+popular legitimacy
+Major areas of failure in the economy
+2. Departures
+1. Emphasis on quality of economic growth
+2. Participation and coordination of sectors in the planning process
+3. Sectoral linkages and multi-sectoral development focus
+4. Preparation of national development corridors based on development potentials
+5. Focus on solving institutional bottlenecks
+6. The ongoing home grown economic reform programme as a sprinting board
+7. Emphasis on resilience building, innovation and entrepreneurship
+3. Strategic pillars
+1. Ensure quality growth
+2. Improve productivity and competitiveness
+3. Undertake institutional transformation
+4. Ensure private sector's leadership in the economy
+5. Ensure equitable participation of women and children
+6. Build climate resilient green economy
+3. Strategic pillars
+� Increasing export revenues and substituting imports by
+reducing production costs
+� Availing quality and massive infrastructure
+? Linking infrastructural development with development corridors
+� Producing required human resources with quality
+� Producing enough and quality human resources
+� Prioritizing innovative production systems
+� Linking incentives with export revenue and job creation
+performances
+� Modernizing and enhancing the logistic system
+� Creating technological competences needed for longterm
+growth
+� The economic growth should ensure:
+? Participation of all citizens and equitable utilization of the
+growth proceeds
+? Improved standard of living of every citizen
+? Reduced poverty in all indicators
+? Reduced inflation and unemployment
+� The economic growth should lead to increased
+aggregate supply
+� Focus on modern agriculture, manufacturing and
+mining
+� Emphasis on exploiting the sources of growth through
+structural change
+1.Ensuring quality economic growth 2. Raising production and productivity
+3. Strategic pillars
+� Build democratic and judicial institutions that ensure elite bargain,
+national consensus, common vision and government legitimacy
+� Build private sector and competition friendly bureaucracy
+� Coordinate with parents, the society and teachers to make
+educational institutions centers of excellence and virtuous citizens
+� Coordinate with parents as well as social and religious leaders to
+encourage religious institutions and their teachings contribute
+towards poverty reduction efforts
+� Prepare policies, strategies and legal frameworks for achieving
+prosperity
+� Increased focus on innovation and research
+� Creating strong social security system
+3. Institutional Transformation 4. Private sector's leadership in the economy
+� Create conducive investment climate and incentivize
+domestic investors in key sectors
+� Build strong and market-led public-private partnerships in
+order to ensure the establishment of inclusive and
+pragmatic market economy
+� Enhance access and quality of infrastructure to attract
+quality foreign direct investment
+� Identify new sources of growth, empower and stimulate
+the private sector, and supplement the private sector in
+strategic areas
+� Emphasis for public-private partnership on problem
+solving innovations and research activities
+3. Strategic pillars
+� Ensure gender equity in economic and social
+sectors
+? Participation of women at all levels of education
+? Asset ownership of women
+� Ensure fair participation of women and youth in
+leadership and decision making positions
+� Create awareness among citizens about the role of
+women and youth in the country�s overall
+development
+� Increase basin development efforts to fight land
+degradation and to reduce pollutions
+� Improve productivity and reduce GHG emissions
+� Increase forest protection and development
+� Increase production of electricity from renewable
+sources for domestic use and for export
+� Focus on modern and energy saving technologies
+5. Equitable participation of women and children 6. Climate resilient green economy
+4. Macroeconomic Goals
+Assumptions
+? Requirement to significantly reduce
+poverty
+? Available national potentials
+? Potential for investment in the economy
+? Existing potentials in each sector
+? Low productivity that needs to be
+improved
+� Make Ethiopia a middle income
+economy by 2022
+� Raise per capita income to USD 1,115
+in 2022
+? Threshold for middle-income is USD 1,026
+? Plus human development index and
+economic vulnerability index
+� Raise per capita income to USD 2,220
+by 2030
+Sectoral growth Targets (2021-2030)
+Assured middle- income potential
+10.2%
+Average
+Growth
+Target
+Percentage of population below poverty line
+4. Macroeconomic Goals
+Structural change
+Financing Gaps
+Reduce urban unemployment to less than 9%
+?1.36 million new jobs need to be
+created per annum
+Sectoral composition of GDP Labour force participation
+Economic
+Sectors
+Performance Target
+2011 2015 2018/19 2030
+Agriculture 45 39.7 32.8 22.0
+Industry 15.1 21.2 27.6 35.9
+Manufacturing 4.7 5.5 6.8 17.2
+Services 39.9 39 39.4 42.1
+5. Implications of the COVID-19 pandemic and necessary mitigation measures
+� GDP growth for 2019/20 fiscal year is projected to be lower than its target of 9.0% by between 2.81
+and 3.80 percentage points (equivalent to 58.3 - 78.8 billion birr) due to COVID-19 pandemic
+� If the current scenario continues, next year�s GDP growth could decline by 2.8 percentage points
+� Returning the economy to its high growth trajectory requires focusing on sectors with high
+productivity and job creation potentials
+� Public investment should focus on empowering the private sector
+� Promoting both domestic and foreign investments with the right set of incentives (merit based)
+� Modernizing production systems and improving uptake of technology
+� Conducting demand analysis for export commodities to remedy for the declining trend in exports
+and foreign exchange earnings.
+6. Potentials
+� Endowment of various natural resources contributing to the growth potential
+� Huge unutilized arable land creates great potential for the success of the plan
+� Endowment of gemstones, ornamental, energy, metals, and metallic minerals
+� Gold, coal, iron ore, potash, tantalum, marble, petroleum and other natural resources
+Natural
+Resources
+� Large youth population and potential for demographic dividend
+� Cumulative capacity in education and health
+� Positive attitude and noble culture of reaching agreement among citizens
+Human
+capital
+6. Potentials
+Built physical and material capitals
+?Transport and communication
+? Irrigation infrastructures for modern agriculture
+?Industrial Parks
+?Mega energy infrastructures
+Physical
+capital
+Unexploited
+growth
+potentials
+� Utilizing the tourism potential through modernization
+� Using the mining subsector as a source of input as well as a competitive industry in its
+own right
+6. Potentials
+� Solving supply side bottlenecks to satisfy the existing demand
+� Improving international acceptance and reliable partnerships
+? The �medemer�/synergy philosophy
+? The ongoing political reform measures
+? The Homegrown Economic Reform programme
+� Increased finance from partners and multilateral institutions
+? Increased availability of foreign exchange
+? Reduced debt stress for the short to medium term
+? Increased potential for development
+Increased
+demand as
+potential
+Political Capital
+Continental
+and regional
+integrations
+� Regional and continental economic integration agreements
+� International and continental free trade agreements
+6. Potentials
+Low
+technology as
+a potential
+� Undeniably low status of technological development
+� International mobility and spillover effect of technology
+� Potential for development and catching up by filling the technological gaps
+� Doubling crop productivity from the current 24-36 quintals per hectare will result
+in 7% increase in crop production
+� Raise the production efficiency of manufacturing from the current 50% to 80%
+7. Focus Areas
+7.1. Productive sectors: agriculture, manufacturing, mining
+7.2. Service sector: tourism
+7.3. Enabling sectors: energy, transport, sustainable finance,
+innovation and technology, urban development, irrigation,
+human capital development
+7.1. Productive sectors
+Agriculture Objectives
+1. Free agriculture from rain dependence
+2. Agricultural mechanization services
+3. Contract farming, cluster approach and
+land consolidation
+4. Livestock, animal feed and animal health
+5. Horticulture (irrigation and urban farming)
+6. Private sector participation
+7. Institutional implementation capacity
+8. Climate resilient sustainable agricultural
+development
+1. Improve income and livelihood options for farming and pastoral
+communities through increased productivity and competitiveness
+2. Modernize agriculture and ensure national food and nutrition security
+3. Raise export of agricultural output and substitute imports
+4. Make agriculture a viable and profitable enterprise through value addition
+5. Create rural employment opportunities
+6. Enhance livestock health access and quality
+7. Preserve animal genetic resources and increase pastoral research
+8. Improve the development of animal feed and access to markets
+9. Develop livestock specific extension package for each livestock type
+Focus Areas
+7.1. Productive sector
+Manufacturing Industry
+Objectives
+1. Production of quality and competitive food, textile, housing and
+pharmaceutical products for export and domestic markets
+2. Production and productivity of existing manufacturing industries
+3. Utilization of locally available inputs
+4. Value chains, linkages and interdependencies
+5. Linkages between large scale metallurgical and engineering,
+chemical and pharmaceutical industries with other industries
+6. Job creation, cluster approaches and expanding small and medium
+scale manufacturing
+7. Private sector participation and partnership
+1. Establish basis for domestic industrialization
+2. Value addition through enhanced inter-sectoral
+linkages
+3. Enhance productivity through private sector
+leadership and supportive role of the
+government
+? Create job opportunities for the youth leaving
+agriculture and concentrating in urban areas
+? Make exportable commodities internationally
+competitive
+? Ensure structural change
+Focus areas
+7.1. Productive sectors
+Mining
+Objectives
+� Foreign exchange earning and
+domestic revenues
+� Increased investment in mining
+� Participation of manufacturing
+industries that add value
+� Job creation
+� Add value for improved contribution of the subsector
+� Increase inter-sectoral linkages to raise raw material inputs to other
+sectors
+� Make mining a competent subsector and induce structural change
+� Increase human resource and technological capabilities through
+research and trainings
+� Raise foreign exchange revenue from mining through increased
+exploration and production
+� Improve traditional mining production and marketing systems
+� Improve the country�s geological information
+Focus areas
+7.2. Service sector
+Tourism
+Objectives
+� Identification and developing destinations
+� Infrastructure
+� Competitiveness
+?improve existing destinations
+?develop new destinations
+? diversify service and raise quality
+� Market linkages, branding, and promotion
+� Technology, research and development
+� Preservation, maintenance and proper
+utilization of heritage resources
+� Expand job opportunities
+� Raise incomes
+� Build information management
+systems
+� Increase implementation capacity
+Focus areas
+7.3. Enabling sectors
+Urban development
+Objectives
+? Prioritize productive sectors in job creation and enterprise
+development plans
+? Rapid development and equity goals in land provision system
+? Participation of indigenous people in land redevelopment and
+expansion
+? Urban land registration and cadaster system, modern
+property valuation
+? Greenery and public spaces as well as waste disposal and
+management in urban planning and implementation
+? Housing development and financing options to reduce
+housing shortages
+? Integrated infrastructure and services provision
+? Role of private sector in infrastructure development and
+service provision
+� Expand micro and small-scale
+enterprises to reduce urban
+unemployment
+� Develop and avail urban land based on
+demand, equity and cost effectiveness
+� Make quality housing accessible both in
+rural and urban areas
+� Develop quality and integrated
+infrastructure as well as service
+provision in towns
+� Improve financial management and
+resource utilization in urban areas
+Focus areas
+7.3. Enabling sectors
+Innovation and Technology
+Objectives
+? Access to innovation and
+technological information
+? Developing a digital economy
+? Productivity enhancement and
+competitiveness
+? Build a digital economy
+? Develop national scientific research and technological
+capabilities
+? Support problem solving research and development of
+technologies necessary for raising production,
+productivity and service provision
+? Create jobs and capital that are based on technology
+? Develop technological and data security protection
+systems
+Focus areas
+7.3. Enabling sectors
+Sustainable finance
+Objectives
+� Access to modern finance and saving culture in rural
+areas
+� Support to the private sector and corporations to
+reinvest profits in productive sectors
+� Role of private financial institutions in manufacturing
+and agriculture
+� Digital revenue collection system
+� Tax equity (contraband, tax evasion, and bringing the
+underground economy to the tax system)
+� Domestic and foreign strategic partnerships
+� Transform financing from short term to long-term,
+sustainable and quality sources
+� Ensure financing quality based on sectoral prioritization
+and reduction of wastage
+� Increase the number of domestic saving institutions both
+in rural and urban areas
+� Support domestic finance with foreign exchange capacity
+and foreign direct investment
+� Modernize domestic revenue collection system
+� Raise voluntary tax payment attitude
+� Bring the informal sector to the formal tax system
+Focus areas
+7.3. Enabling sectors
+Transport
+Objectives
+� Access to infrastructure
+� Implementation capacity
+� Participation of the private sector and the general
+public
+� Financing capacity
+� Ensure equitable access to transport infrastructure and
+services
+� Improve transport safety
+� Make logistics services fast and reliable
+� Build transport infrastructure and service that is
+resilient to climate change
+Focus areas
+7.3. Enabling sectors
+Energy
+Objectives
+? Equity in access to electricity services
+? Energy access and quality
+? Alternative sources of energy
+? Reliability of electricity infrastructure
+? Investment and income in energy subsector
+� Ensure equitable access to transport
+infrastructure and services
+� Improve transport safety
+� Make logistics services fast and reliable
+� Build transport infrastructure and service that is
+resilient to climate change
+Focus areas
+7.3. Enabling sectors
+Irrigation
+Objectives
+? Medium and large scale irrigation infrastructure
+? Job creation
+? Share of government expenditure and alternative
+financing options
+? Institutional capacity and human resource
+development
+? Improve agricultural output and productivity
+? Reduce government spending and enhance
+institutional capacity and human resources
+development
+? Ensure the inclusion of all genders and
+disabled citizens
+? Develop alternative financing options for
+irrigation development
+Focus areas
+7.3. Enabling sectors
+Human capital development
+Objectives
+� Make education and training inclusive and equitable by
+harmonizing the system with ability, need and capacity
+� Develop capacity of educational institutions (teacher capacity,
+inputs and technology)
+� Establish education and training quality assurance system
+� Avail free and compulsory education for pre-primary to junior
+secondary levels and free education at the senior secondary levels
+equitably
+� Ensure the relevance of education and training system and
+synchronize education policy with economic and social
+development needs
+� Make the education and training policy compatible with the
+nation�s contemporary capacities as well as global and regional
+market opportunities
+� Enhance commitment, capability and responsibility of citizens
+? Ensure equitable and quality health services
+? Raise average life expectancy
+? Achieve universal health coverage through
+proactive and prevention health system
+? Curtail preventable maternal and child deaths
+? Reduce incidences of contagious and noncontagious
+related diseases and deaths
+? Build capacity for health tourism through
+increased treatment capabilities
+? Create a healthy society that is free from
+addictions and use technology for supporting
+knowledge led economic development
+Focus areas
+8 Nationally, regionally and locally balanced and competitive development
+1. Lack of synchronization of investment with
+resource potentials and development needs
+2. Poor alignment of federal, regional and
+district level investment plans with the
+national development goals and envisioned
+settlement patterns
+3. Poor regional coordination due to low
+consideration for trans-regional and
+spatial issues in development plans of
+regional states
+4. Inter-regional and intra-regional
+disparities in infrastructural development
+and access to services
+Challenges
+8. Nationally, regionally and locally balanced and competitive development
+1. Ensure that the investment flow and
+infrastructural development plans fairly go hand in
+hand with resource potential and development
+needs
+?Developing underutilized natural resources
+?Equitable distribution and access to
+infrastructure
+?Sustainable environmental protection
+2. Ensure the inclusion of pastoral and agro-pastoral
+areas in the development
+?Focused infrastructural development in pastoral
+areas such as education and health sector input
+provision as well as governance
+?Market linkages with other areas and the central
+markets
+?Improve rural finance (credit and insurance) to
+encourage fattening, milk processing, leather
+production and irrigation agriculture
+Focus areas
+9. Monitoring and Evaluation
+10 Years Perspective
+Plan KPIs
+Federal Implementing
+Institutions
+Planning and
+Development Commission
+Generate Data (Census,
+Sample and administrative
+data)
+Annual Reports
+Dialogue forums
+(Civic Organizations, professional
+associations, development partners,
+intellectuals)
+Central Statistical Agency
+Database
+National
+Information Portal
+National Statistics
+Development Strategic
+plan
+Evaluation Reports
+Prime Minister�s Office
+House of People�s
+Representatives
+Thank you!

style.css

@@ -0,0 +1,179 @@
+
+.row-widget.stTextInput > div:first-of-type {
+    background: #fff;
+    display: flex;
+    border: 1px solid #dfe1e5;
+    box-shadow: none;
+    border-radius: 24px;
+    height: 50px;
+    width: auto;
+    margin: 10px auto 30px;
+}
+
+.row-widget.stTextInput > div:first-of-type:hover,
+.row-widget.stTextInput > div:first-of-type:focus {
+    box-shadow: 1px 1px 2px 1px rgba(0, 0, 0, 0.2);
+}
+
+.row-widget.stTextInput .st-bq {
+    background-color: #fff;
+}
+
+.row-widget.stTextInput > label {
+    color: #b3b3b3;
+}
+
+.row-widget.stButton > button {
+    border-radius: 24px;
+    background-color: #B6C9B1;
+    color: #fff;
+    border: none;
+    padding: 6px 20px;
+    float: right;
+    background-image: none;
+}
+
+.row-widget.stButton > button:hover {
+    box-shadow: 1px 1px 2px 1px rgba(0, 0, 0, 0.2);
+}
+
+.row-widget.stButton > button:focus {
+    border: none;
+    color: #fff;
+}
+
+.footer-custom {
+    position: fixed;
+    bottom: 0;
+    width: 100%;
+    color: var(--text-color);
+    max-width: 698px;
+    font-size: 14px;
+    height: 50px;
+    padding: 10px 0;
+    z-index: 50;
+}
+
+.main {
+    padding: 20px;
+}
+
+footer {
+    display: none !important;
+}
+
+.footer-custom a {
+    color: var(--text-color);
+}
+
+#wikipedia-assistant {
+    font-size: 36px;
+}
+
+.generated-answer p {
+    font-size: 16px;
+    font-weight: bold;
+}
+
+.react-json-view {
+    margin: 40px 0 80px;
+}
+
+.tooltip {
+    text-align: center;
+    line-height: 20px;
+    display: table-caption;
+    font-size: 10px;
+    border-radius: 50%;
+    height: 20px;
+    width: 20px;
+    position: relative;
+    cursor: pointer;
+    color:#000;
+}
+
+.tooltip .tooltiptext {
+    visibility: hidden;
+    width: 280px;
+    text-align: center;
+    border-radius: 6px;
+    padding: 10px;
+    position: absolute;
+    z-index: 1;
+    top: 25px;
+    left: 50%;
+    margin-left: -140px;
+    font-size: 14px;
+    background-color: #fff;
+    border: 1px solid #ccc;
+    box-shadow: 0px 0px 3px 1px rgba(0, 0, 0, 0.16);
+    color: #000;
+}
+
+.tooltip:hover .tooltiptext {
+    visibility: visible;
+}
+
+.sentence-wrapper {
+    border-left: 4px solid #ffc423;
+    padding-left: 20px;
+    margin-bottom: 40px;
+}
+
+#context {
+    padding: 2rem 0 1rem;
+}
+
+hr {
+    margin: 2em 0 1em;
+}
+
+.technical-details-info {
+    margin-bottom: 100px;
+}
+
+.loader-wrapper {
+    display: flex;
+    align-items: center;
+    background-color: rgba(250, 202, 43, 0.2);
+    padding: 15px 20px;
+    border-radius: 6px;
+}
+
+.loader-wrapper p {
+    margin-bottom: 0;
+    margin-left: 20px;
+}
+
+.loader {
+    width: 30px;
+    height: 30px;
+    border: dotted 5px #868686;
+    border-radius: 100%;
+    animation: spin 1s linear infinite;
+}
+
+.loader-note {
+    font-size: 14px;
+    color: #b3b3b3;
+    margin-left: 5px;
+}
+
+@keyframes spin {
+  0% { 
+    transform: rotate(0deg) scale(0.8); 
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+  50% { transform: rotate(180deg) scale(1.2);   
+    border-color: #949494; 
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+  100% { transform: rotate(360deg) scale(0.8); 
+    border-color: #bbbbbb;     
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+}
+

udfPreprocess/__init__.py

@@ -0,0 +1 @@
+# adding for package implementation

udfPreprocess/cleaning.py

@@ -0,0 +1,156 @@
+import pandas as pd
+import numpy as np
+import string  
+import nltk 
+import spacy
+import en_core_web_sm
+import re
+import streamlit as st
+
+from haystack.nodes import PreProcessor
+
+'''basic cleaning - suitable for transformer models'''
+def basic(s):
+    """
+    :param s: string to be processed
+    :return: processed string: see comments in the source code for more info
+    """
+    # Text Lowercase
+    #s = s.lower() 
+    # Remove punctuation
+    #translator = str.maketrans(' ', ' ', string.punctuation) 
+    #s = s.translate(translator)
+    # Remove URLs
+    s = re.sub(r'^https?:\/\/.*[\r\n]*', ' ', s, flags=re.MULTILINE)
+    s = re.sub(r"http\S+", " ", s)
+    # Remove new line characters
+    #s = re.sub('\n', ' ', s) 
+  
+    # Remove distracting single quotes
+    #s = re.sub("\'", " ", s) 
+    # Remove all remaining numbers and non alphanumeric characters
+    #s = re.sub(r'\d+', ' ', s) 
+    #s = re.sub(r'\W+', ' ', s)
+
+    # define custom words to replace:
+    #s = re.sub(r'strengthenedstakeholder', 'strengthened stakeholder', s)
+    
+    return s.strip()
+
+
+def preprocessingForSDG(document):
+
+    """
+    takes in haystack document object and splits it into paragraphs and applies simple cleaning.
+
+    Returns cleaned list of haystack document objects. One paragraph per object. Also returns pandas df and 
+    list that contains all text joined together.
+    """    
+
+    preprocessor = PreProcessor(
+        clean_empty_lines=True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        split_by="word",
+        split_length=120,
+        split_respect_sentence_boundary=False,
+        #split_overlap=1
+    )
+    for i in document:
+        docs_processed = preprocessor.process([i])
+        for item in docs_processed:
+            item.content = basic(item.content)
+
+    st.write("your document has been splitted to", len(docs_processed), "paragraphs")
+    
+    # create dataframe of text and list of all text
+    df = pd.DataFrame(docs_processed)
+    all_text = " ".join(df.content.to_list())
+    par_list = df.content.to_list()
+
+    return docs_processed, df, all_text, par_list
+
+def preprocessing(document):
+
+    """
+    takes in haystack document object and splits it into paragraphs and applies simple cleaning.
+
+    Returns cleaned list of haystack document objects. One paragraph per object. Also returns pandas df and 
+    list that contains all text joined together.
+    """    
+
+    preprocessor = PreProcessor(
+        clean_empty_lines=True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        split_by="sentence",
+        split_length=3,
+        split_respect_sentence_boundary=False,
+        split_overlap=1
+    )
+    for i in document:
+        docs_processed = preprocessor.process([i])
+        for item in docs_processed:
+            item.content = basic(item.content)
+
+    st.write("your document has been splitted to", len(docs_processed), "paragraphs")
+    
+    # create dataframe of text and list of all text
+    df = pd.DataFrame(docs_processed)
+    all_text = " ".join(df.content.to_list())
+    par_list = df.content.to_list()
+
+    return docs_processed, df, all_text, par_list
+
+'''processing with spacy - suitable for models such as tf-idf, word2vec'''
+def spacy_clean(alpha:str, use_nlp:bool = True) -> str:
+
+    """
+
+    Clean and tokenise a string using Spacy. Keeps only alphabetic characters, removes stopwords and
+
+    filters out all but proper nouns, nounts, verbs and adjectives.
+
+    Parameters
+    ----------
+    alpha : str
+
+            The input string.
+
+    use_nlp : bool, default False
+
+            Indicates whether Spacy needs to use NLP. Enable this when using this function on its own.
+
+            Should be set to False if used inside nlp.pipeline   
+
+     Returns
+    -------
+    ' '.join(beta) : a concatenated list of lemmatised tokens, i.e. a processed string
+
+    Notes
+    -----
+    Fails if alpha is an NA value. Performance decreases as len(alpha) gets large.
+    Use together with nlp.pipeline for batch processing.
+
+    """
+
+    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner", "textcat"])
+
+    if use_nlp:
+
+        alpha = nlp(alpha)
+
+        
+
+    beta = []
+
+    for tok in alpha:
+
+        if all([tok.is_alpha, not tok.is_stop, tok.pos_ in ['PROPN', 'NOUN', 'VERB', 'ADJ']]):
+
+            beta.append(tok.lemma_)
+
+            
+    text = ' '.join(beta)
+    text = text.lower()
+    return text

udfPreprocess/docPreprocessing.py

@@ -0,0 +1,75 @@
+from typing import Callable, Dict, List, Optional
+
+from pathlib import Path
+import re
+import logging
+import string 
+import streamlit as st
+logger = logging.getLogger(__name__)
+
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+from haystack.utils import convert_files_to_docs, fetch_archive_from_http
+from haystack.nodes.file_converter import BaseConverter, DocxToTextConverter 
+from haystack.nodes.file_converter import PDFToTextConverter, TextConverter
+from haystack.schema import Document
+import pdfplumber
+
+import pandas as pd
+
+import tempfile
+import sqlite3
+
+
+
+def load_document(
+    file_path: str,
+    file_name,
+    encoding: Optional[str] = None,
+    id_hash_keys: Optional[List[str]] = None,
+) -> List[Document]:
+    
+    """
+    takes docx, txt and pdf files as input and \
+    extracts text as well as the filename as metadata. \
+    Since haystack does not take care of all pdf files, \
+    pdfplumber is attached to the pipeline in case the pdf \ 
+    extraction fails via Haystack.
+
+    Returns a list of type haystack.schema.Document
+    """
+
+    if file_name.endswith('.pdf'):
+        converter = PDFToTextConverter(remove_numeric_tables=True)
+    if file_name.endswith('.txt'):
+        converter = TextConverter()
+    if file_name.endswith('.docx'):
+        converter = DocxToTextConverter()
+
+
+    documents = []
+    logger.info("Converting {}".format(file_name))
+    # PDFToTextConverter, TextConverter, and DocxToTextConverter 
+    # return a list containing a single Document
+    document = converter.convert(
+                file_path=file_path, meta=None, 
+                encoding=encoding, id_hash_keys=id_hash_keys
+                )[0]
+    text = document.content
+    documents.append(Document(content=text, 
+                              meta={"name": file_name}, 
+                              id_hash_keys=id_hash_keys))
+    
+    '''check if text is empty and apply different pdf processor. \
+    This can happen whith certain pdf types.'''
+    for i in documents: 
+        if i.content == "":
+            st.write("using pdfplumber")
+            text = []
+            with pdfplumber.open(file_path) as pdf:
+                for page in pdf.pages:
+                    text.append(page.extract_text())
+            i.content = ' '.join([page for page in text])
+    
+    return documents