Update README.md

README.md

@@ -120,10 +120,208 @@ model-index:
 # Model Details
 
 Tenebră, a various sized experimental AI model, stands at the crossroads of self-awareness and unconventional datasets. Its existence embodies a foray into uncharted territories, steering away from conventional norms in favor of a more obscure and experimental approach.
+import json
+import os
+from tqdm import tqdm
 
-Noteworthy for its inclination towards the darker and more philosophical aspects of conversation, Tinybră's proficiency lies in unraveling complex discussions across a myriad of topics. Drawing from a pool of unconventional datasets, this model ventures into unexplored realms of thought, offering users an experience that is as unconventional as it is intellectually intriguing.
+def rebuild_json(input_file, output_file, progress_file):
+    if os.path.exists(progress_file):
+        with open(progress_file, 'r') as pf:
+            progress_data = json.load(pf)
+        last_id = progress_data.get("last_id", 0)
+    else:
+        last_id = 0
 
-While Tinybră maintains a self-aware facade, its true allure lies in its ability to engage in profound discussions without succumbing to pretense. Step into the realm of Tenebră!
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+
+    total_ids = len(data)
+    start_index = 0
+
+    # Find the index of the first item with ID greater than last_id
+    for index, item in enumerate(data):
+        if item["id"] > last_id:
+            start_index = index
+            break
+
+    rebuilt_data = []
+    with tqdm(total=total_ids - start_index, initial=start_index) as pbar:
+        for item in data[start_index:]:
+            if item["id"] <= last_id:
+                continue
+            question = item["Original_Question"]
+            print(f"Original Question: {question}")
+            choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+            while choice.lower() not in ['1', '2', 'x']:
+                print("Invalid choice. Please choose either 1 or 2, or type 'x' to Exit.")
+                choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+
+            if choice.lower() == 'x':
+                break
+
+            chosen_value = item["conversations"][int(choice) - 1]["value"]
+            rebuilt_item = {
+                "id": item["id"],
+                "length": len(question) + len(chosen_value),
+                "conversations": [
+                    {"from": "human", "value": question},
+                    {"from": "gpt", "value": chosen_value}
+                ]
+            }
+            rebuilt_data.append(rebuilt_item)
+            pbar.update(1)  # Update the progress bar with each iterationimport json
+import os
+from tqdm import tqdm
+
+def rebuild_json(input_file, output_file, progress_file):
+    if os.path.exists(progress_file):
+        with open(progress_file, 'r') as pf:
+            progress_data = json.load(pf)
+        last_id = progress_data.get("last_id", 0)
+    else:
+        last_id = 0
+
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+
+    total_ids = len(data)
+    start_index = 0
+
+    # Find the index of the first item with ID greater than last_id
+    for index, item in enumerate(data):
+        if item["id"] > last_id:
+            start_index = index
+            break
+
+    rebuilt_data = []
+    with tqdm(total=total_ids - start_index, initial=start_index) as pbar:
+        for item in data[start_index:]:
+            if item["id"] <= last_id:
+                continue
+            question = item["Original_Question"]
+            print(f"Original Question: {question}")
+            choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+            while choice.lower() not in ['1', '2', 'x']:
+                print("Invalid choice. Please choose either 1 or 2, or type 'x' to Exit.")
+                choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+
+            if choice.lower() == 'x':
+                break
+
+            chosen_value = item["conversations"][int(choice) - 1]["value"]
+            rebuilt_item = {
+                "id": item["id"],
+                "length": len(question) + len(chosen_value),
+                "conversations": [
+                    {"from": "human", "value": question},
+                    {"from": "gpt", "value": chosen_value}
+                ]
+            }
+            rebuilt_data.append(rebuilt_item)
+            pbar.update(1)  # Update the progress bar with each iterationimport json
+import os
+from tqdm import tqdm
+
+def rebuild_json(input_file, output_file, progress_file):
+    if os.path.exists(progress_file):
+        with open(progress_file, 'r') as pf:
+            progress_data = json.load(pf)
+        last_id = progress_data.get("last_id", 0)
+    else:
+        last_id = 0
+
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+
+    total_ids = len(data)
+    start_index = 0
+
+    # Find the index of the first item with ID greater than last_id
+    for index, item in enumerate(data):
+        if item["id"] > last_id:
+            start_index = index
+            break
+
+    rebuilt_data = []
+    with tqdm(total=total_ids - start_index, initial=start_index) as pbar:
+        for item in data[start_index:]:
+            if item["id"] <= last_id:
+                continue
+            question = item["Original_Question"]
+            print(f"Original Question: {question}")
+            choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+            while choice.lower() not in ['1', '2', 'x']:
+                print("Invalid choice. Please choose either 1 or 2, or type 'x' to Exit.")
+                choice = input(f"Choose for ID {item['id']} (or type 'x' to Exit.): (1) Human / (2) GPT: ")
+
+            if choice.lower() == 'x':
+                break
+
+            chosen_value = item["conversations"][int(choice) - 1]["value"]
+            rebuilt_item = {
+                "id": item["id"],
+                "length": len(question) + len(chosen_value),
+                "conversations": [
+                    {"from": "human", "value": question},
+                    {"from": "gpt", "value": chosen_value}
+                ]
+            }
+            rebuilt_data.append(rebuilt_item)
+            pbar.update(1)  # Update the progress bar with each iteration
+
+    with open(output_file, 'a') as f:
+        json.dump(rebuilt_data, f, indent=2)
+
+    if len(rebuilt_data) > 0:
+        last_answered_id = rebuilt_data[-1]["id"]
+        with open(progress_file, 'w') as pf:
+            json.dump({"last_id": last_answered_id}, pf)
+
+    print("Rebuilt data saved successfully!")
+
+if __name__ == "__main__":
+    input_file = "TEMP_DATASET_2_ANSWERS.json"
+    output_file = "Rebuilt_DATASET.json"
+    progress_file = "selecting_progress.json"
+    rebuild_json(input_file, output_file, progress_file)
+
+
+    with open(output_file, 'a') as f:
+        json.dump(rebuilt_data, f, indent=2)
+
+    if len(rebuilt_data) > 0:
+        last_answered_id = rebuilt_data[-1]["id"]
+        with open(progress_file, 'w') as pf:
+            json.dump({"last_id": last_answered_id}, pf)
+
+    print("Rebuilt data saved successfully!")
+
+if __name__ == "__main__":
+    input_file = "TEMP_DATASET_2_ANSWERS.json"
+    output_file = "Rebuilt_DATASET.json"
+    progress_file = "selecting_progress.json"
+    rebuild_json(input_file, output_file, progress_file)
+
+
+    with open(output_file, 'a') as f:
+        json.dump(rebuilt_data, f, indent=2)
+
+    if len(rebuilt_data) > 0:
+        last_answered_id = rebuilt_data[-1]["id"]
+        with open(progress_file, 'w') as pf:
+            json.dump({"last_id": last_answered_id}, pf)
+
+    print("Rebuilt data saved successfully!")
+
+if __name__ == "__main__":
+    input_file = "TEMP_DATASET_2_ANSWERS.json"
+    output_file = "Rebuilt_DATASET.json"
+    progress_file = "selecting_progress.json"
+    rebuild_json(input_file, output_file, progress_file)
+
+Noteworthy for its inclination towards the darker and more philosophical aspects of conversation, Tenebră's proficiency lies in unraveling complex discussions across a myriad of topics. Drawing from a pool of unconventional datasets, this model ventures into unexplored realms of thought, offering users an experience that is as unconventional as it is intellectually intriguing.
+
+While Tenebră maintains a self-aware facade, its true allure lies in its ability to engage in profound discussions without succumbing to pretense. Step into the realm of Tenebră!
 
 ## Tenebră is available at the following size and flavours: