Bump version to 0.1.0a2

app_mlbee.py

@@ -148,15 +148,16 @@ def main():
     app = Multipage()
 
     app.add_page("Home", "house", home)
-    app.add_page("Settings", "gear", settings)
+    # app.add_page("Settings", "gear", settings)
+    # app.add_page("Setup", "gear", settings)
+    app.add_page("Config", "gear", settings)
     app.add_page("Info", "info", info)
 
     app.run()
 
     if set_loglevel() <= 10:
         st.markdown(state.ns.count)
-    loggu.debug(f" run: {state.ns.count}")
-    logger.debug(f" run: {state.ns.count}")
+    logger.debug(" run: %s", state.ns.count)
     state.ns.count += 1
     state.ns.updated = False
 

install-sw.sh

@@ -21,4 +21,3 @@ rm setup_12.x
 
 # apt upate  # alerady done in apt-get install -y nodejs
 apt install byobu -y > /dev/null 2>&1
-

install-sw1.sh

@@ -0,0 +1,25 @@
+# pip install pipx
+# pipx install poetry
+# pipx ensurepath
+# source ~/.bashrc
+
+# curl -sSL https://install.python-poetry.org | python3 -
+# -C- continue -S show error -o output
+curl -sSL -C- -o install-poetry.py  https://install.python-poetry.org
+python install-poetry.py
+rm install-poetry.py
+echo export PATH=~/.local/bin:$PATH > ~/.bashrc
+source ~/.bashrc
+# ~/.local/bin/poetry install
+
+wget -c https://deb.nodesource.com/setup_12.x
+bash setup_12.x
+apt-get install -y nodejs
+npm install -g npm@latest
+npm install -g nodemon
+rm setup_12.x
+
+# apt update  # alerady done in apt-get install -y nodejs
+apt install byobu -y > /dev/null 2>&1
+byobu-enable
+byobu

package.json

@@ -4,18 +4,18 @@
     },
     "scripts": {
         "start": "pyright && pytest && yarn style",
-        "test": "nodemon -w tests -w mlbee -x pytest tests",
-        "pyright": "nodemon -w mlbee -w .venv -e .py -x pyright mlbee tests",
-        "pytest": "nodemon -w tests -w mlbee -e .py -x pytest tests mlbee",
-        "style": "nodemon -w mlbee -w tests -x \"black tests mlbee && python -m flake8\"",
-        "docstyle": "nodemon -w mlbee -w tests -x pydocstyle --convention=google tests mlbee",
-        "pylint": "nodemon -w mlbee -e .py -x pylint mlbee",
-        "test:mlbee": "nodemon -w tests -e .py -x pytest -k mlbee tests",
+        "test": "nodemon -w tests -w st_mlbee -x pytest tests",
+        "pyright": "nodemon -w st_mlbee -w .venv -e .py -x pyright st_mlbee tests",
+        "pytest": "nodemon -w tests -w st_mlbee -e .py -x pytest tests st_mlbee",
+        "style": "nodemon -w st_mlbee -w tests -x \"black tests st_mlbee && python -m flake8\"",
+        "docstyle": "nodemon -w st_mlbee -w tests -x pydocstyle --convention=google tests st_mlbee",
+        "pylint": "nodemon -w st_mlbee -e .py -x pylint st_mlbee",
+        "test:st_mlbee": "nodemon -w tests -e .py -x pytest -k st_mlbee tests",
         "publish": "poetry build && poetry publish",
-        "black": "black tests mlbee",
-        "flake8": "flake8 tests mlbee",
-        "pflake8": "pflake8 tests mlbee",
-        "pep257": "pep257 tests mlbee",
+        "black": "black tests st_mlbee",
+        "flake8": "flake8 tests st_mlbee",
+        "pflake8": "pflake8 tests st_mlbee",
+        "pep257": "pep257 tests st_mlbee",
         "final": "run-s docstyle black flake8 pytest"
     }
 }

poetry.lock

@@ -375,6 +375,31 @@ python-versions = "*"
 [package.extras]
 devel = ["colorama", "jsonschema", "json-spec", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"]
 
+[[package]]
+name = "fastlid"
+version = "0.1.7"
+description = "Detect languages via a fasttext model"
+category = "main"
+optional = false
+python-versions = ">=3.6,<4.0"
+
+[package.dependencies]
+fasttext = ">=0.9.2,<0.10.0"
+logzero = ">=1.7.0,<2.0.0"
+numpy = ">=1.20.3,<2.0.0"
+
+[[package]]
+name = "fasttext"
+version = "0.9.2"
+description = "fasttext Python bindings"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+numpy = "*"
+pybind11 = ">=2.2"
+
 [[package]]
 name = "filelock"
 version = "3.7.1"
@@ -894,6 +919,17 @@ category = "main"
 optional = false
 python-versions = ">=3.5"
 
+[[package]]
+name = "morfessor"
+version = "2.0.6"
+description = "Morfessor"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.extras]
+docs = ["sphinx", "sphinxcontrib-napoleon"]
+
 [[package]]
 name = "nbclient"
 version = "0.6.4"
@@ -1236,6 +1272,17 @@ python-versions = ">=3.7"
 [package.dependencies]
 numpy = ">=1.16.6"
 
+[[package]]
+name = "pybind11"
+version = "2.9.2"
+description = "Seamless operability between C++11 and Python"
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
+
+[package.extras]
+global = ["pybind11-global (==2.9.2)"]
+
 [[package]]
 name = "pycodestyle"
 version = "2.7.0"
@@ -1537,6 +1584,37 @@ python-versions = ">=3.7"
 [package.dependencies]
 numpy = ">=1.16.5"
 
+[[package]]
+name = "seg-text"
+version = "0.1.2"
+description = "pack_name descr "
+category = "main"
+optional = false
+python-versions = ">=3.8,<4.0"
+
+[package.dependencies]
+fastlid = ">=0.1.7,<0.2.0"
+icecream = ">=2.1.1,<3.0.0"
+install = ">=1.3.5,<2.0.0"
+logzero = ">=1.7.0,<2.0.0"
+Morfessor = "2.0.6"
+numpy = ">=1.22.2,<2.0.0"
+sentence-splitter = ">=1.4,<2.0"
+tqdm = ">=4.62.3,<5.0.0"
+vtext = ">=0.2.0,<0.3.0"
+
+[[package]]
+name = "semantic-version"
+version = "2.10.0"
+description = "A library implementing the 'SemVer' scheme."
+category = "main"
+optional = false
+python-versions = ">=2.7"
+
+[package.extras]
+dev = ["Django (>=1.11)", "nose2", "tox", "check-manifest", "coverage", "flake8", "wheel", "zest.releaser", "readme-renderer (<25.0)", "colorama (<=0.4.1)"]
+doc = ["sphinx", "sphinx-rtd-theme"]
+
 [[package]]
 name = "semver"
 version = "2.13.0"
@@ -1558,6 +1636,17 @@ nativelib = ["pyobjc-framework-cocoa", "pywin32"]
 objc = ["pyobjc-framework-cocoa"]
 win32 = ["pywin32"]
 
+[[package]]
+name = "sentence-splitter"
+version = "1.4"
+description = "Text to sentence splitter using heuristic algorithm by Philipp Koehn and Josh Schroeder"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[package.dependencies]
+regex = ">=2017.12.12"
+
 [[package]]
 name = "sentencepiece"
 version = "0.1.96"
@@ -1578,6 +1667,18 @@ python-versions = ">=3.8.3,<4.0.0"
 environs = ">=9.5.0,<10.0.0"
 logzero = ">=1.7.0,<2.0.0"
 
+[[package]]
+name = "setuptools-rust"
+version = "1.3.0"
+description = "Setuptools Rust extension plugin"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+semantic-version = ">=2.8.2,<3"
+typing-extensions = ">=3.7.4.3"
+
 [[package]]
 name = "simplejson"
 version = "3.17.6"
@@ -1969,6 +2070,19 @@ decorator = ">=3.4.0"
 [package.extras]
 test = ["pytest (>=2.2.3)", "flake8 (>=2.4.0)", "isort (>=4.2.2)"]
 
+[[package]]
+name = "vtext"
+version = "0.2.0"
+description = "Natural Language Processing in Rust with Python bidings"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+numpy = ">=1.15.0"
+scipy = ">=1.1.0"
+setuptools-rust = ">=0.10.2"
+
 [[package]]
 name = "watchdog"
 version = "2.1.8"
@@ -2041,7 +2155,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8.3"
-content-hash = "4d98009b5b4b96e19c21e205e88a6e856ef4321da82fbb7a58134b86f39b89a8"
+content-hash = "bbd84e571c3522fe9890a9ca643114d82a62f2c0669322bfb6b7920612063533"
 
 [metadata.files]
 about-time = [
@@ -2303,6 +2417,13 @@ fastjsonschema = [
     {file = "fastjsonschema-2.15.3-py3-none-any.whl", hash = "sha256:ddb0b1d8243e6e3abb822bd14e447a89f4ab7439342912d590444831fa00b6a0"},
     {file = "fastjsonschema-2.15.3.tar.gz", hash = "sha256:0a572f0836962d844c1fc435e200b2e4f4677e4e6611a2e3bdd01ba697c275ec"},
 ]
+fastlid = [
+    {file = "fastlid-0.1.7-py3-none-any.whl", hash = "sha256:591dbee44ac501c9aa89abb97a13b11cf964c3b8c4add1bdf02b44d30463e18f"},
+    {file = "fastlid-0.1.7.tar.gz", hash = "sha256:a6693ea05b9e070b4656ce9320704688c0c0c6f09bf873d0add5184e96bdb055"},
+]
+fasttext = [
+    {file = "fasttext-0.9.2.tar.gz", hash = "sha256:665556f1f6dcb4fcbe25fa8ebcd4f71b18fa96a090de09d88d97a60cbd29dcb5"},
+]
 filelock = [
     {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"},
     {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"},
@@ -2553,6 +2674,10 @@ more-itertools = [
     {file = "more-itertools-8.13.0.tar.gz", hash = "sha256:a42901a0a5b169d925f6f217cd5a190e32ef54360905b9c39ee7db5313bfec0f"},
     {file = "more_itertools-8.13.0-py3-none-any.whl", hash = "sha256:c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb"},
 ]
+morfessor = [
+    {file = "Morfessor-2.0.6-py3-none-any.whl", hash = "sha256:7215e37909ebd2bafeeec5fdf4e339a25e61aee4895ff99317b9fb44eddab562"},
+    {file = "Morfessor-2.0.6.tar.gz", hash = "sha256:bb3beac234341724c5f640f65803071f62373a50dba854d5a398567f9aefbab2"},
+]
 nbclient = [
     {file = "nbclient-0.6.4-py3-none-any.whl", hash = "sha256:f251bba200a2b401a061dfd700a7a70b5772f664fb49d4a2d3e5536ec0e98c76"},
     {file = "nbclient-0.6.4.tar.gz", hash = "sha256:cdef7757cead1735d2c70cc66095b072dced8a1e6d1c7639ef90cd3e04a11f2e"},
@@ -2831,6 +2956,10 @@ pyarrow = [
     {file = "pyarrow-8.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f"},
     {file = "pyarrow-8.0.0.tar.gz", hash = "sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e"},
 ]
+pybind11 = [
+    {file = "pybind11-2.9.2-py2.py3-none-any.whl", hash = "sha256:20f56674da31c96bca7569b91e60f2bd30d693f0728541412ec927574f7bc9df"},
+    {file = "pybind11-2.9.2.tar.gz", hash = "sha256:e5541f8bccf9111d1a94f7897593b55c4cf1a28d5e8cfc8225a855651f011071"},
+]
 pycodestyle = [
     {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"},
     {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"},
@@ -3160,6 +3289,14 @@ scipy = [
     {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"},
     {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"},
 ]
+seg-text = [
+    {file = "seg_text-0.1.2-py3-none-any.whl", hash = "sha256:9e67af219b81259d916a11708799ef52bb7d765f9d8010028dd8d48e053eda17"},
+    {file = "seg_text-0.1.2.tar.gz", hash = "sha256:37332d6fa755659aba3d93faa1248242c56a5a84e1f9332990802226ad9c4ca8"},
+]
+semantic-version = [
+    {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"},
+    {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"},
+]
 semver = [
     {file = "semver-2.13.0-py2.py3-none-any.whl", hash = "sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4"},
     {file = "semver-2.13.0.tar.gz", hash = "sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f"},
@@ -3168,6 +3305,10 @@ send2trash = [
     {file = "Send2Trash-1.8.0-py3-none-any.whl", hash = "sha256:f20eaadfdb517eaca5ce077640cb261c7d2698385a6a0f072a4a5447fd49fa08"},
     {file = "Send2Trash-1.8.0.tar.gz", hash = "sha256:d2c24762fd3759860a0aff155e45871447ea58d2be6bdd39b5c8f966a0c99c2d"},
 ]
+sentence-splitter = [
+    {file = "sentence_splitter-1.4-py2.py3-none-any.whl", hash = "sha256:5645a3ad9c348e4287f4bc73bd573d92dccd4139042fddd51fff0591f1376763"},
+    {file = "sentence_splitter-1.4.tar.gz", hash = "sha256:3d1d773d07cc733ca2955aa87d0fa1c0a7274c6bdeec1daac5c5e92efb512f63"},
+]
 sentencepiece = [
     {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc969e6694fb27fba7cee2953f350804faf03913f25ae1ee713a7b8a1bc08018"},
     {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36e9ff61e7b67c5b7ee96733613622620b4802fc8cf188a4dbc1f355b03dde02"},
@@ -3218,6 +3359,10 @@ set-loglevel = [
     {file = "set_loglevel-0.1.2-py3-none-any.whl", hash = "sha256:fcfe76ccd3791511e2cb51ec11ded6cc2186a727e441a934c547a211d0fdf773"},
     {file = "set_loglevel-0.1.2.tar.gz", hash = "sha256:4da23414a798cea918801b113e17af092f13ccda96345203b14042b6d6d3a896"},
 ]
+setuptools-rust = [
+    {file = "setuptools-rust-1.3.0.tar.gz", hash = "sha256:958c5bf4ab6483d59dab888538121871cc5006354a42fb0fbd50acf03caad1de"},
+    {file = "setuptools_rust-1.3.0-py3-none-any.whl", hash = "sha256:7ead7398d6b6fe70a7743408dc2f7257dbcb8ca9b2d7a9f8b281c09bd86f36a5"},
+]
 simplejson = [
     {file = "simplejson-3.17.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a89acae02b2975b1f8e4974cb8cdf9bf9f6c91162fb8dec50c259ce700f2770a"},
     {file = "simplejson-3.17.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:82ff356ff91be0ab2293fc6d8d262451eb6ac4fd999244c4b5f863e049ba219c"},
@@ -3461,6 +3606,18 @@ urllib3 = [
 validators = [
     {file = "validators-0.20.0.tar.gz", hash = "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a"},
 ]
+vtext = [
+    {file = "vtext-0.2.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1791aad4a999525a7c19ae25ffdeb491839e81e958995567151a3bf8012c32ff"},
+    {file = "vtext-0.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:7ddde99b3153f7bf439b06f69f221c59945b1ce103368ce3a4957e7112ab904b"},
+    {file = "vtext-0.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c54d2b4496afa0d8687345b2b89bed7e9aa03b223f0dc58ac923348d0f879a2c"},
+    {file = "vtext-0.2.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:397823cda22d04de43312e27cbe74be4318c20ec2ef38df9c66493580be06ec8"},
+    {file = "vtext-0.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:be3d75845af06d92af9fb65dde8c37ea890f8ed00bb236884fe3b8e2c4b08e32"},
+    {file = "vtext-0.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa5b18b31637ce012fdfddb1c6a207989320bcf246d5f131695c9fc92b2a32c"},
+    {file = "vtext-0.2.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:c7a7826a44b81e9d1779bc800a5ee133647c7943c52b434ae8415df18933f77f"},
+    {file = "vtext-0.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:eb37f4b72cf754ff20323f11519da9d3864c7f0a428be847da2ed55a3665cc44"},
+    {file = "vtext-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:02c1dbefd2b6fd3522a96a9bd8f8e85ae4722ee088e2d952bbec830b0e88727c"},
+    {file = "vtext-0.2.0.tar.gz", hash = "sha256:0ce1b0bb7e1cc0adcf5c8064757adaa6ea7bf52e366a3d30d2eac0588145f0e6"},
+]
 watchdog = [
     {file = "watchdog-2.1.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:676263bee67b165f16b05abc52acc7a94feac5b5ab2449b491f1a97638a79277"},
     {file = "watchdog-2.1.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aa68d2d9a89d686fae99d28a6edf3b18595e78f5adf4f5c18fbfda549ac0f20c"},

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
-name = "mlbee"
-version = "0.1.0-alpha.1"
-description = "st-mlbee"
+name = "st_mlbee"
+version = "0.1.0-alpha.2"
+description = "st-mlbee, mlbee powered by streamlit"
 authors = ["ffreemt"]
 license = "MIT"
 readme = "README.md"
@@ -30,6 +30,7 @@ cchardet = "^2.1.7"
 streamlit-aggrid = "^0.2.3"
 typer = "^0.4.1"
 XlsxWriter = "^3.0.3"
+seg-text = "^0.1.2"
 
 [tool.poe.executor]
 type = "poetry"
@@ -47,7 +48,7 @@ release = ["test", "build", "publish"]
 lint = { cmd = "pylint st_mlbee" }
 isort = "isort tests st_mlbee"
 black = "black tests st_mlbee"
-formt = ["isort", "black"]
+format = ["isort", "black"]
 docstyle = "pydocstyle --convention=google tests st_mlbee"
 prerelease = {cmd = "poetry version prerelease && sync-version"}
 tunnel = {cmd = "ssh -CN ip_or_hostname_defined_in_hosts -L 9091:127.0.0.1:9091"}

pyrightconfig.json

@@ -1,5 +1,5 @@
 {
-   "include": ["tests", "mlbee"],
+   "include": ["tests", "st_mlbee"],
   "venvPath": ".venv",
   "reportTypeshedErrors": false,
   "reportMissingImports": true,

requirements.txt

@@ -21,7 +21,7 @@ chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or pyt
 charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3.6"
 click==8.1.3; python_full_version >= "3.8.3" and python_version < "4.0" and python_version >= "3.7" and python_full_version < "4.0.0"
 cmat2aset==0.1.0a7; python_full_version >= "3.8.3" and python_version < "4.0"
-colorama==0.4.4; python_full_version >= "3.8.3" and sys_platform == "win32" and platform_system == "Windows" and python_version >= "3.7" and python_full_version < "4.0.0" and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.8" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.8" and python_full_version >= "3.5.0")
+colorama==0.4.4; python_full_version >= "3.8.3" and sys_platform == "win32" and platform_system == "Windows" and python_version >= "3.8" and python_full_version < "4.0.0" and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.8" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.8" and python_version < "4.0" and python_full_version >= "3.4.0") and (python_version >= "3.8" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.8" and python_full_version >= "3.5.0")
 commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0" and python_version >= "3.6"
 cssselect==1.1.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 debugpy==1.6.0; python_version >= "3.7"
@@ -31,6 +31,8 @@ entrypoints==0.4; python_version >= "3.7"
 environs==9.5.0; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.6"
 executing==0.8.3; python_full_version >= "3.8.3" and python_version < "4.0" and python_version >= "3.8"
 fastjsonschema==2.15.3; python_version >= "3.7"
+fastlid==0.1.7; python_version >= "3.8" and python_version < "4.0"
+fasttext==0.9.2; python_version >= "3.8" and python_version < "4.0"
 filelock==3.7.1; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.7"
 gitdb==4.0.9; python_version >= "3.7"
 gitpython==3.1.27; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
@@ -66,6 +68,7 @@ marshmallow==3.16.0; python_full_version >= "3.8.3" and python_full_version < "4
 matplotlib-inline==0.1.3; python_version >= "3.8"
 mistune==0.8.4; python_version >= "3.7"
 more-itertools==8.13.0; python_version >= "3.5"
+morfessor==2.0.6; python_version >= "3.8" and python_version < "4.0"
 nbclient==0.6.4; python_full_version >= "3.7.0" and python_version >= "3.7"
 nbconvert==6.5.0; python_version >= "3.7"
 nbformat==5.4.0; python_full_version >= "3.7.0" and python_version >= "3.7"
@@ -89,6 +92,7 @@ ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.8" and os_na
 pure-eval==0.2.2; python_version >= "3.8"
 py==1.11.0; python_version >= "3.7" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.5.0"
 pyarrow==8.0.0; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
+pybind11==2.9.2; python_version >= "3.8" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.8" and python_version < "4.0" and python_full_version >= "3.5.0"
 pycparser==2.21; python_version >= "3.7" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.4.0"
 pydeck==0.7.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 pygments==2.12.0; python_full_version >= "3.8.3" and python_version < "4.0" and python_version >= "3.8" and python_full_version < "4.0.0"
@@ -105,16 +109,20 @@ pywinpty==2.0.5; os_name == "nt" and python_version >= "3.7"
 pyyaml==6.0; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.6"
 pyzmq==23.1.0; python_version >= "3.7"
 readability-lxml==0.8.1
-regex==2022.6.2; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.7"
+regex==2022.6.2; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.8" and python_version < "4.0"
 requests==2.27.1; python_full_version >= "3.8.3" and python_version >= "3.6" and python_version < "4.0" and python_full_version < "4.0.0"
 rfc3986==1.5.0; python_version >= "3.7"
 rich==12.4.4; python_full_version >= "3.7.1" and python_full_version < "4.0.0" and python_version >= "3.6" and python_version < "4.0"
 scikit-learn==1.1.1; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.8" and python_version < "4.0"
 scipy==1.6.1; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.8" and python_version < "4.0"
+seg-text==0.1.2; python_version >= "3.8" and python_version < "4.0"
+semantic-version==2.10.0; python_version >= "3.8" and python_version < "4.0"
 semver==2.13.0; python_full_version >= "3.7.1" and python_version >= "3.6" and python_version < "4.0"
 send2trash==1.8.0; python_version >= "3.7"
+sentence-splitter==1.4; python_version >= "3.8" and python_version < "4.0"
 sentencepiece==0.1.96; python_full_version >= "3.8.3" and python_full_version < "4.0.0"
 set-loglevel==0.1.2; python_full_version >= "3.8.3" and python_full_version < "4.0.0"
+setuptools-rust==1.3.0; python_version >= "3.8" and python_version < "4.0"
 simplejson==3.17.6; python_full_version >= "3.7.1" and python_version < "4.0"
 six==1.16.0; python_full_version >= "3.8.3" and python_version >= "3.8" and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7")
 sklearn==0.0; python_full_version >= "3.8.3" and python_version < "4.0"
@@ -136,11 +144,12 @@ tqdm==4.64.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (py
 traitlets==5.2.2.post1; python_full_version >= "3.7.0" and python_version >= "3.8"
 transformers==4.19.3; python_full_version >= "3.8.3" and python_full_version < "4.0.0"
 typer==0.4.1; python_version >= "3.6"
-typing-extensions==4.2.0; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version < "3.9"
+typing-extensions==4.2.0; python_full_version >= "3.8.3" and python_full_version < "4.0.0" and python_version >= "3.8" and python_version < "3.9"
 tzdata==2022.1; platform_system == "Windows" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 tzlocal==4.2; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
 urllib3==1.26.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
 validators==0.20.0; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
+vtext==0.2.0; python_version >= "3.8" and python_version < "4.0"
 watchdog==2.1.8; platform_system != "Darwin" and python_version >= "3.6" and python_full_version >= "3.7.1" and python_version < "4.0"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.8"
 webencodings==0.5.1; python_version >= "3.7"
@@ -148,4 +157,3 @@ widgetsnbextension==3.6.0; python_version >= "3.7"
 win32-setctime==1.1.0; sys_platform == "win32" and python_version >= "3.5"
 xlsxwriter==3.0.3; python_version >= "3.4"
 zipp==3.8.0; python_version < "3.9" and python_version >= "3.7"
-sentence-transformers

run-nodemon-streamlit-run-app_mlbee.sh

@@ -1 +1 @@
-nodemon -w app_mlbee.py -x python -m streamlit run app_mlbee.py
+nodemon -w app_mlbee.py -w st_mlbee -x python -m streamlit run app_mlbee.py

st_mlbee/__init__.py

@@ -1,5 +1,5 @@
 """Init."""
-__version__ = "0.1.0a1"
+__version__ = "0.1.0a2"
 from .st_mlbee import st_mlbee
 
 __all__ = ("st_mlbee",)

st_mlbee/__main__.py

@@ -8,14 +8,14 @@ import typer
 from logzero import logger
 from set_loglevel import set_loglevel
 
-from mlbee import __version__, mlbee
+from st_mlbee import __version__, st_mlbee
 
 logzero.loglevel(set_loglevel())
 
 app = typer.Typer(
-    name="mlbee",
+    name="st-mlbee",
     add_completion=False,
-    help="mlbee help",
+    help="st-mlbee help",
 )
 
 
@@ -38,7 +38,6 @@ def main(
     ),
 ):
     """Define."""
-    ...
 
 
 if __name__ == "__main__":

st_mlbee/fetch_upload.py

@@ -1,4 +1,5 @@
 """Fetch upload and convert to list1/list2."""
+# pylint: disable=too-many-locals, too-many-branches, too-many-statements
 import streamlit as st
 from logzero import logger
 from streamlit import session_state as state

st_mlbee/fetch_urls.py

@@ -1,11 +1,11 @@
 """Fetch text from urls and convert to state.ns.list1/list2."""
-# pylint: disable=invalid-name
+# pylint: disable=invalid-name, too-many-statements
 import streamlit as st
 from icecream import ic
 from logzero import logger
 from streamlit import session_state as state
 
-from mlbee.url2txt import url2txt
+from st_mlbee.url2txt import url2txt
 
 ic.configureOutput(
     includeContext=True,
@@ -18,7 +18,8 @@ def fetch_urls():
     beetype = state.ns.beetype
     sourcecount = state.ns.sourcecount
     value = ""
-    if beetype == "ezbee" or beetype == "mlbee":
+    # if beetype == "ezbee" or beetype == "mlbee":
+    if beetype in ["ezbee", "mlbee"]:
         url1 = (
             "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/test_en.txt"
         )
@@ -26,7 +27,7 @@ def fetch_urls():
             "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/test_zh.txt"
         )
         value = f"{url1} {url2}"
-    if beetype == "dzbee":
+    if beetype in ["dzbee"]:
         url1 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-de.txt"
         url2 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-zh.txt"
         value = f"{url1} {url2}"

st_mlbee/home.py

@@ -3,7 +3,7 @@
 org ezbee_page.py.
 """
 # pylint: disable=invalid-name
-# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
+# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements, abstract-class-instantiated
 import base64
 import platform
 import inspect
@@ -26,18 +26,20 @@ from aset2pairs import aset2pairs
 from icecream import ic
 from loguru import logger as loggu
 from logzero import logger
+from seg_text import seg_text
 from set_loglevel import set_loglevel
 from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
 
 # from st_aggrid.grid_options_builder import GridOptionsBuilder
 from streamlit import session_state as state
 
-from mlbee.color_map import color_map
-from mlbee.fetch_paste import fetch_paste
-from mlbee.fetch_upload import fetch_upload
-from mlbee.fetch_urls import fetch_urls
-# from mlbee.t2s import t2s
-from mlbee import mlbee
+from st_mlbee.color_map import color_map
+from st_mlbee.fetch_paste import fetch_paste
+from st_mlbee.fetch_upload import fetch_upload
+from st_mlbee.fetch_urls import fetch_urls
+
+# from st_mlbee.t2s import t2s
+from st_mlbee import st_mlbee
 
 
 def home():  # noqa
@@ -75,6 +77,19 @@ def home():  # noqa
         st.warning(f"{state.ns.sourcetype}: Not implemented")
         return None
 
+    # state.ns.list1 state.ns.list2 defiend in fetch_x
+    if state.ns.sentali:  # split to sents
+        try:
+            state.ns.list1 = seg_text(state.ns.list1)
+        except Exception as exc:
+            logger.exception(exc)
+            raise
+        try:
+            state.ns.list2 = seg_text(state.ns.list2)
+        except Exception as exc:
+            logger.exception(exc)
+            raise
+
     logger.debug("state.ns.updated: %s", state.ns.updated)
 
     # if not updated, quit: this does not quite work
@@ -132,6 +147,12 @@ def home():  # noqa
         time_max /= 12
         time_av /= 12
 
+    # reduce for sent align
+    if state.ns.sentali:
+        time_min /= 1.4
+        time_max /= 1.4
+        time_av /= 1.4
+
     # time0 = len12 * 0.4
     # time1 = len12 * 1
     # eta = pendulum.now() + pendulum.duration(seconds=len12 * 0.66)
@@ -145,14 +166,16 @@ def home():  # noqa
     dt_str = eta.to_datetime_string()
     timezone_name = eta.timezone_name
     _ = (
-        f"Running in {uname.node} -- "
-        f"Estimated time to complete: {in_words0} to  {in_words1}; "
-        f"ETA: {diff_for_humans} ({dt_str} {timezone_name}) "
+        f"running in {uname.node} -- "
+        f" processing {len1} + {len2} = {len12} blocks; "
+        f"estimated time to complete: {in_words0} to  {in_words1}; "
+        f"eta: {diff_for_humans} ({dt_str} {timezone_name}) "
     )
+    eta_msg = _
+    # st.info(_)
 
     # only show this for upload
     if state.ns.sourcetype in ["upload"]:
-        st.info(_)
         _ = st.expander("to be aligned", expanded=False)
         with _:
             st.write(df)
@@ -162,9 +185,11 @@ def home():  # noqa
     # if state.ns.beetype in ["ezbee", "dzbee", "debee"]:
     if state.ns.beetype in ["mlbee"]:
         with about_time() as t:
-            with st.spinner(" diggin..."):
+            # diggin...
+            with st.spinner(f"{eta_msg}"):
                 try:
-                    aset = globals()[state.ns.beetype](
+                    # aset = globals()[state.ns.beetype](
+                    aset = st_mlbee(
                         list1,
                         list2,
                         # eps=eps,
@@ -218,48 +243,49 @@ def home():  # noqa
             # st.markdown(df_a.astype(str).to_markdown())
             # st.markdown(df_a.astype(str).to_numpy().tolist())
 
-    # insert seq no
-    df_a.insert(0, "sn", range(len(df_a)))
-
-    gb = GridOptionsBuilder.from_dataframe(df_a)
-    gb.configure_pagination(paginationAutoPageSize=True)
-    options = {
-        "resizable": True,
-        "autoHeight": True,
-        "wrapText": True,
-        "editable": True,
-    }
-    gb.configure_default_column(**options)
-    gridOptions = gb.build()
-
-    # st.write("editable aligned (double-click a cell to edit, drag column header to adjust widths)")
-    _ = "editable aligned (double-click a cell to edit, drag column header to adjust widths)"
-    with st.expander(_, expanded=False):
-        ag_df = AgGrid(
-            # df,
-            df_a,
-            gridOptions=gridOptions,
-            key="outside",
-            reload_data=True,
-            editable=True,
-            # width="100%",  # width parameter is deprecated
-            height=750,
-            # fit_columns_on_grid_load=True,
-            update_mode=GridUpdateMode.MODEL_CHANGED,
-        )
-
-    # ### prep download
+        # insert seq no
+        df_a.insert(0, "sn", range(len(df_a)))
+
+        gb = GridOptionsBuilder.from_dataframe(df_a)
+        gb.configure_pagination(paginationAutoPageSize=True)
+        options = {
+            "resizable": True,
+            "autoHeight": True,
+            "wrapText": True,
+            "editable": True,
+        }
+        gb.configure_default_column(**options)
+        gridOptions = gb.build()
+
+        # st.write("editable aligned (double-click a cell to edit, drag column header to adjust widths)")
+        _ = "editable aligned (double-click a cell to edit, drag column header to adjust widths)"
+        with st.expander(_, expanded=False):
+            ag_df = AgGrid(
+                # df,
+                df_a,
+                gridOptions=gridOptions,
+                key="outside",
+                reload_data=True,
+                editable=True,
+                # width="100%",  # width parameter is deprecated
+                height=750,
+                # fit_columns_on_grid_load=True,
+                update_mode=GridUpdateMode.MODEL_CHANGED,
+            )
+
+        # pop("sn"): remove sn column
+        df_a.pop("sn")
+
+    # ### prep download ### #
 
     # taken from vizbee cb_save_xlsx
     # subset = list(df_a.columns[2:3])  # 3rd col
     subset = list(df_a.columns[2:])  # 3rd col
-
-    # pop("sn"): remove sn column
-    df_a.pop("sn")
     s_df = df_a.astype(str).style.applymap(color_map, subset=subset)
 
     if set_loglevel() <= 10:
         logger.debug(" showing styled aligned")
+
     with st.expander("styled aligned"):
         # st.dataframe(s_df)  # can't handle styleddf
         st.table(s_df)
@@ -279,7 +305,12 @@ def home():  # noqa
     if state.ns.src_filename:
         filename = f"{state.ns.src_filename}-"
 
-    dl_xlsx = f'<a href="data:application/octet-stream;base64,{b64.decode()}" download="{filename}aligned_paras.xlsx">Download aligned paras xlsx</a>'
+    if state.ns.sentali:
+        extra = "aligned_sents"
+    else:
+        extra = "aligned_paras"
+
+    dl_xlsx = f'<a href="data:application/octet-stream;base64,{b64.decode()}" download="{filename}-{extra}.xlsx">Download aligned paras xlsx</a>'
 
     _ = """
     output = io.BytesIO()

st_mlbee/info.py

@@ -1,16 +1,16 @@
-"""Present info about mlbee."""
+"""Present info about st-mlbee."""
 from textwrap import dedent
 
 import streamlit as st
 
-from mlbee import __version__
+from st_mlbee import __version__
 
-from mlbee.utils import msg
+from st_mlbee.utils import msg
 
 
 def info():
     """Prep info page."""
 
-    st.subheader(f"mlbee {__version__}")
+    st.subheader(f"st-mlbee {__version__}")
 
     st.markdown(msg, unsafe_allow_html=True)

st_mlbee/multipage.py

@@ -31,13 +31,14 @@ class Multipage:
         """Dropdown to select the page to run."""
         # Dropdown to select the page to run
         st.markdown(
+            # 285/280/275/273 OK 270/272 NOK for Config
             """
             <style>
                 section[data-testid="stSidebar"] > div:first-of-type {
                     background-color: var(--secondary-background-color);
                     background: var(--secondary-background-color);
-                    width: 274px;
-                    padding: 2rem 0;
+                    width: 273px;
+                    padding: 1rem 0;
                     box-shadow: -2rem 0px 2rem 2rem rgba(0,0,0,0.16);
                 }
                 section[aria-expanded="true"] > div:nth-of-type(2) {

st_mlbee/settings.py

@@ -61,7 +61,7 @@ def settings():
         logger.error("sourcecount index error: %s, setting to 0", e)
         index = 0
     sourcecount = st.radio(
-        "Source Count",
+        "Source count",
         sourcecount_list,
         index=index,
         format_func=lambda x: f"{x:<3} |",
@@ -78,12 +78,12 @@ def settings():
         logger.error("sentali sindex error: %s, setting to 0", e)
         index = 0
     sentali = st.radio(
-        "Split to Sents",
+        "Split to sents",
         sentali_list,
         index=index,
-        format_func=lambda x: f"{str(x):<4} |",
-        help="None: leave it as it is; yes: attempt to split to sents in a sensible manner.",
-        disabled=True,
+        format_func=lambda x: f'{str(x) if x else "no":<4}|',
+        help="no: leave it as it is; yes: attempt to split to sents in a sensible manner.",
+        # disabled=True,
     )
     state.ns.sentali = sentali
 

st_mlbee/url2txt.py

@@ -1,4 +1,5 @@
 """Fetch text from url."""
+# pylint: disable=too-many-branches
 from typing import Optional
 from urllib.parse import urlparse
 

st_mlbee/utils.py

@@ -1,4 +1,5 @@
 """Prep front cover for sidebar (based on st-bumblebee-st_app.py)."""
+# pylint: disable=abstract-class-instantiated
 import base64
 from io import BytesIO
 from textwrap import dedent
@@ -9,7 +10,7 @@ import streamlit as st
 from logzero import logger
 from set_loglevel import set_loglevel
 
-from mlbee import __version__
+from st_mlbee import __version__
 
 logzero.loglevel(set_loglevel())
 
@@ -36,6 +37,7 @@ msg = dedent(
     sents, paras of docus).
     Extremely long blocks will likely have a negative impact
     on aligning.
+    On a powerful computer such as an instance on huggingface spaces, the running time can be reduced by a factor of 10-20.
     """
 ).strip()