Add oasst-sft-6-llama-30b XORs

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,86 @@
 ---
 license: other
 ---
+
+# OpenAssistant LLaMa-Based Models
+
+Due to the license attached to LLaMa models by Meta AI it is not possible to directly distribute LLaMa-based models. Instead we provide XOR weights for the OA models.
+
+Thanks to Mick for writing the `xor_codec.py` script which enables this process
+
+## The Process
+
+Note: This process applies to `oasst-sft-6-llama-30b` model. The same process can be applied to other models in future, but the checksums will be different..
+
+To use OpenAssistant LLaMa-Based Models, you need to have a copy of the original LLaMa model weights and add them to a `llama` subdirectory here.
+
+Ensure your LLaMa 30B checkpoint matches the correct md5sums:
+
+```
+f856e9d99c30855d6ead4d00cc3a5573  consolidated.00.pth
+d9dbfbea61309dc1e087f5081e98331a  consolidated.01.pth
+2b2bed47912ceb828c0a37aac4b99073  consolidated.02.pth
+ea0405cdb5bc638fee12de614f729ebc  consolidated.03.pth
+4babdbd05b8923226a9e9622492054b6  params.json
+```
+
+These can be converted to HuggingFace Transformers-compatible weights using the script available [here](https://github.com/huggingface/transformers/blob/28f26c107b4a1c5c7e32ed4d9575622da0627a40/src/transformers/models/llama/convert_llama_weights_to_hf.py).
+
+**Important**: It was tested with git version transformers 4.28.0.dev0 (git hash: **28f26c107b4a1c5c7e32ed4d9575622da0627a40**). Make sure the package tokenizers 0.13.3 is installed. Use of different versions may result in broken outputs.
+
+```
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python python convert_llama_weights_to_hf.py --input_dir ~/llama/  --output_dir ~/llama30b_hf/ --model_size 30B
+```
+
+Run `find -type f -exec md5sum "{}" + > checklist.chk` in the conversion target directory. This should produce a `checklist.chk` with exactly the following content if your files are correct:
+
+```
+d0e13331c103453e9e087d59dcf05432  ./pytorch_model-00001-of-00007.bin
+29aae4d31a0a4fe6906353001341d493  ./pytorch_model-00002-of-00007.bin
+b40838eb4e68e087b15b3d653ca1f5d7  ./pytorch_model-00003-of-00007.bin
+f845ecc481cb92b8a0586c2ce288b828  ./pytorch_model-00004-of-00007.bin
+f3b13d089840e6caf22cd6dd05b77ef0  ./pytorch_model-00005-of-00007.bin
+12e0d2d7a9c00c4237b1b0143c48a05e  ./pytorch_model-00007-of-00007.bin
+1348f7c8bb3ee4408b69305a10bdfafb  ./pytorch_model-00006-of-00007.bin
+aee09e21813368c49baaece120125ae3  ./generation_config.json
+eeec4125e9c7560836b4873b6f8e3025  ./tokenizer.model
+598538f18fed1877b41f77de034c0c8a  ./config.json
+fdb311c39b8659a5d5c1991339bafc09  ./tokenizer.json
+b77e99aa2ddc3df500c2b2dc4455a6af  ./pytorch_model.bin.index.json
+edd1a5897748864768b1fab645b31491  ./tokenizer_config.json
+6b2e0a735969660e720c27061ef3f3d3  ./special_tokens_map.json
+```
+
+Once you have LLaMa weights in the correct format, you can apply the XOR decoding:
+
+```
+python xor_codec.py oasst-sft-6-llama-30b/ oasst-sft-6-llama-30b-xor/ llama30b_hf/
+```
+
+You should expect to see one warning message during execution:
+
+`Exception when processing 'added_tokens.json'`
+
+This is normal. If similar messages appear for other files, something has gone wrong.
+
+Now run `find -type f -exec md5sum "{}" + > checklist.chk` in the output directory (here `oasst-sft-6-llama-30b`). You should get a file with exactly these contents:
+
+```
+970e99665d66ba3fad6fdf9b4910acc5  ./pytorch_model-00007-of-00007.bin
+659fcb7598dcd22e7d008189ecb2bb42  ./pytorch_model-00003-of-00007.bin
+ff6e4cf43ddf02fb5d3960f850af1220  ./pytorch_model-00001-of-00007.bin
+27b0dc092f99aa2efaf467b2d8026c3f  ./added_tokens.json
+aee09e21813368c49baaece120125ae3  ./generation_config.json
+740c324ae65b1ec25976643cda79e479  ./pytorch_model-00005-of-00007.bin
+f7aefb4c63be2ac512fd905b45295235  ./pytorch_model-00004-of-00007.bin
+eeec4125e9c7560836b4873b6f8e3025  ./tokenizer.model
+369df2f0e38bda0d9629a12a77c10dfc  ./pytorch_model-00006-of-00007.bin
+27b9c7c8c62db80e92de14724f4950f3  ./config.json
+deb33dd4ffc3d2baddcce275a00b7c1b  ./tokenizer.json
+76d47e4f51a8df1d703c6f594981fcab  ./pytorch_model.bin.index.json
+ed59bfee4e87b9193fea5897d610ab24  ./tokenizer_config.json
+130f5e690becc2223f59384887c2a505  ./special_tokens_map.json
+ae48c4c68e4e171d502dd0896aa19a84  ./pytorch_model-00002-of-00007.bin
+```
+
+If so you have successfully decoded the weights and should be able to use the model with HuggingFace Transformers.

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/added_tokens.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae0cd1d42605f0b35eed61a86a8549b9bc26db2f659f024edec2161b1c382661
+size 133

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb3641cfa9a7a45ae4408105e74bb75e1aab548f16c3a8b1d0909197e61c1909
+size 578

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/generation_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b26ac0994c5ab22e3e73b649058eb16e7878825f9cf9076f7beddf6d352f6fec
+size 137

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00001-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f84110fff294a3ff7933ca753272670f3c5e396e78a8f180905423d1bcd94e92
+size 9818537619

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00002-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0958e77886aad4e111656afb5ccd82b3d1aea852e892bc4c11ec81adfc130493
+size 9958102743

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00003-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1806498954ec6c5d8f5cacf0861cfe822bdaab169827c65d46154d3e5424f49
+size 9896734715

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00004-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:307d03af990b90b56d3cad50e27e69c3de6a703514f66a2abf27862445956a72
+size 9869470481

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00005-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c318332fc85c005f37c8d82ec2fdbea6d6694e51caccf48870edea63b4978b
+size 9869470445

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00006-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cff07e8b3c407a8fe8060c6fbd1fc76e37fbe80a50e10e573ad87273cc34fc7
+size 9958102743

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model-00007-of-00007.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58df43b706b808f196c57b918754d9a19fbc6c0397456e1575647ee48ba4924e
+size 5688116273

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/pytorch_model.bin.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f7249a4ca9881a77b9858a4a027aa5c1d406a4e4fa16b892b2866d3b0e0776
+size 50084

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b52a967a3325e3d7d0b22c2d03a4d392a1a848027350251e25502744c079aebf
+size 213

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68147850c080987172d24ad27a9ba2c65c71b46e248e8ee0f0c4eda90e2ca558
+size 499723

oasst-sft-6-llama-30b-xor/oasst-sft-6-llama-30b-xor/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f1f8cb8184eeed4e429f4f52c7b39f5883757a65688a5775fc061390102966e
+size 277

xor_codec.py

@@ -0,0 +1,83 @@
+import os
+import sys
+import shutil
+import gzip
+import numpy
+from pathlib import Path
+
+def xor_uncompressed(dst, src_payload, src_base, block_size=4096):
+    fp_payload = open(src_payload, 'rb')
+    fp_base = open(src_base, 'rb')
+    with open(dst, 'wb') as fp:
+        while True:
+            buf1 = numpy.array(bytearray(fp_payload.read(block_size)), dtype=numpy.uint8)
+            buf2 = numpy.array(bytearray(fp_base.read(block_size)), dtype=numpy.uint8)
+            padding = len(buf1) - len(buf2)
+            if padding > 0: buf2 = numpy.pad(buf2, (0, padding), 'constant', constant_values=(0,))
+            if padding < 0: buf2 = buf2[:len(buf1)]
+            buf = numpy.bitwise_xor(buf1, buf2)
+            fp.write(buf)
+            if len(buf1) < block_size: break
+    fp_payload.close()
+    fp_base.close()
+
+def xor_encode(dst, src_payload, src_base, block_size=4096):
+    fp_payload = open(src_payload, 'rb')
+    fp_base = open(src_base, 'rb')
+    with gzip.open(dst, 'wb') as fp:
+        while True:
+            buf1 = numpy.array(bytearray(fp_payload.read(block_size)), dtype=numpy.uint8)
+            buf2 = numpy.array(bytearray(fp_base.read(block_size)), dtype=numpy.uint8)
+            padding = len(buf1) - len(buf2)
+            if padding > 0: buf2 = numpy.pad(buf2, (0, padding), 'constant', constant_values=(0,))
+            if padding < 0: buf2 = buf2[:len(buf1)]
+            buf = numpy.bitwise_xor(buf1, buf2)
+            fp.write(buf)
+            if len(buf1) < block_size: break
+    fp_payload.close()
+    fp_base.close()
+
+def xor_decode(dst, src_payload, src_base, block_size=4096):
+    fp_payload = gzip.open(src_payload, 'rb')
+    fp_base = open(src_base, 'rb')
+    with open(dst, 'wb') as fp:
+        while True:
+            buf1 = numpy.array(bytearray(fp_payload.read(block_size)), dtype=numpy.uint8)
+            buf2 = numpy.array(bytearray(fp_base.read(block_size)), dtype=numpy.uint8)
+            padding = len(buf1) - len(buf2)
+            if padding > 0: buf2 = numpy.pad(buf2, (0, padding), 'constant', constant_values=(0,))
+            if padding < 0: buf2 = buf2[:len(buf1)]
+            buf = numpy.bitwise_xor(buf1, buf2)
+            fp.write(buf)
+            if len(buf1) < block_size: break
+    fp_payload.close()
+    fp_base.close()
+
+def xor_dir(dst, src_payload, src_base, decode=True, compress=True):
+    if compress:
+        xor = xor_decode if decode else xor_encode 
+    else:
+        xor = xor_uncompressed
+    Path(dst).mkdir(parents=True, exist_ok=True)
+    shutil.copy(Path(src_payload) / "added_tokens.json", Path(dst) / "added_tokens.json")
+    for path in os.listdir(src_payload):
+        print("[*] Processing '%s'" % path)
+        try:    
+            xor("%s/%s" % (dst, path), "%s/%s" % (src_payload, path), "%s/%s" % (src_base, path))
+        except Exception as e:
+            print("Exception when processing '%s'" % path)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 4:
+        print("Usage: xor.py <DESTINATION> <PAYLOAD SOURCE> <LLAMA SOURCE> [--encode] [--compress]")
+        exit()
+    dst = sys.argv[1]
+    src_payload = sys.argv[2]
+    src_base = sys.argv[3]
+    decode = True
+    compress = False
+    if len(sys.argv) > 4:
+        for arg in sys.argv[4:]:
+            if arg == "--encode": decode = False
+            if arg == "--compress": compress = True
+    xor_dir(dst, src_payload, src_base, decode=decode, compress=compress)