Files changed (2) hide show
  1. gguf-imat-llama-3.py +22 -17
  2. gguf-imat.py +22 -17
gguf-imat-llama-3.py CHANGED
@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
70
  def download_model_repo():
71
  base_dir = os.path.dirname(os.path.abspath(__file__))
72
  models_dir = os.path.join(base_dir, "models")
 
73
  if not os.path.exists(models_dir):
74
  os.makedirs(models_dir)
75
 
@@ -77,27 +78,27 @@ def download_model_repo():
77
  model_name = model_id.split("/")[-1]
78
  model_dir = os.path.join(models_dir, model_name)
79
 
80
- if os.path.exists(model_dir):
81
- print("Model repository already exists. Using existing repository.")
82
-
83
- delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
84
-
85
- imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
86
-
87
- convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
88
 
89
- else:
90
- revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
 
 
91
 
92
- delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
93
 
94
- print("Downloading model repository...")
95
- snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
96
- print("Model repository downloaded successfully.")
97
 
98
- imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
 
 
99
 
100
- convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
101
 
102
  def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
103
  convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
@@ -115,7 +116,11 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
115
  print(f"Original model directory '{model_dir}' deleted.")
116
  else:
117
  print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
 
 
 
118
 
 
119
  imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
120
  imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
121
  imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
@@ -161,4 +166,4 @@ def main():
161
  print("Finished preparing resources.")
162
 
163
  if __name__ == "__main__":
164
- main()
 
70
  def download_model_repo():
71
  base_dir = os.path.dirname(os.path.abspath(__file__))
72
  models_dir = os.path.join(base_dir, "models")
73
+
74
  if not os.path.exists(models_dir):
75
  os.makedirs(models_dir)
76
 
 
78
  model_name = model_id.split("/")[-1]
79
  model_dir = os.path.join(models_dir, model_name)
80
 
81
+ gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
82
+ gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
83
+ imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
84
+ delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
 
 
 
 
85
 
86
+ if os.path.exists(gguf_model_path):
87
+ create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
88
+ else:
89
+ if os.path.exists(model_dir):
90
+ print("Model repository already exists. Using existing repository.")
91
 
92
+ convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
93
 
94
+ else:
95
+ revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
96
 
97
+ print("Downloading model repository...")
98
+ snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
99
+ print("Model repository downloaded successfully.")
100
 
101
+ convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
102
 
103
  def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
104
  convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
 
116
  print(f"Original model directory '{model_dir}' deleted.")
117
  else:
118
  print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
119
+
120
+
121
+ create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
122
 
123
+ def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
124
  imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
125
  imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
126
  imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
 
166
  print("Finished preparing resources.")
167
 
168
  if __name__ == "__main__":
169
+ main()
gguf-imat.py CHANGED
@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
70
  def download_model_repo():
71
  base_dir = os.path.dirname(os.path.abspath(__file__))
72
  models_dir = os.path.join(base_dir, "models")
 
73
  if not os.path.exists(models_dir):
74
  os.makedirs(models_dir)
75
 
@@ -77,27 +78,27 @@ def download_model_repo():
77
  model_name = model_id.split("/")[-1]
78
  model_dir = os.path.join(models_dir, model_name)
79
 
80
- if os.path.exists(model_dir):
81
- print("Model repository already exists. Using existing repository.")
82
-
83
- delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
84
-
85
- imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
86
-
87
- convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
88
 
89
- else:
90
- revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
 
 
91
 
92
- delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
93
 
94
- print("Downloading model repository...")
95
- snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
96
- print("Model repository downloaded successfully.")
97
 
98
- imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
 
 
99
 
100
- convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
101
 
102
  def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
103
  convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
@@ -115,13 +116,17 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
115
  print(f"Original model directory '{model_dir}' deleted.")
116
  else:
117
  print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
 
 
 
118
 
 
119
  imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
120
  imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
121
  imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
122
  if not os.path.exists(imatrix_output_dst):
123
  try:
124
- subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "8"], cwd=gguf_dir)
125
  shutil.move(imatrix_output_src, imatrix_output_dst)
126
  print("imatrix.dat moved successfully.")
127
  except Exception as e:
 
70
  def download_model_repo():
71
  base_dir = os.path.dirname(os.path.abspath(__file__))
72
  models_dir = os.path.join(base_dir, "models")
73
+
74
  if not os.path.exists(models_dir):
75
  os.makedirs(models_dir)
76
 
 
78
  model_name = model_id.split("/")[-1]
79
  model_dir = os.path.join(models_dir, model_name)
80
 
81
+ gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
82
+ gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
83
+ imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
84
+ delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
 
 
 
 
85
 
86
+ if os.path.exists(gguf_model_path):
87
+ create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
88
+ else:
89
+ if os.path.exists(model_dir):
90
+ print("Model repository already exists. Using existing repository.")
91
 
92
+ convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
93
 
94
+ else:
95
+ revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
96
 
97
+ print("Downloading model repository...")
98
+ snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
99
+ print("Model repository downloaded successfully.")
100
 
101
+ convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
102
 
103
  def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
104
  convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
 
116
  print(f"Original model directory '{model_dir}' deleted.")
117
  else:
118
  print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
119
+
120
+
121
+ create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
122
 
123
+ def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
124
  imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
125
  imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
126
  imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
127
  if not os.path.exists(imatrix_output_dst):
128
  try:
129
+ subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "7"], cwd=gguf_dir)
130
  shutil.move(imatrix_output_src, imatrix_output_dst)
131
  print("imatrix.dat moved successfully.")
132
  except Exception as e: