Add DI-sheep

.gitattributes

@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+DI-sheep/service/ckpt_best.pth.tar filter=lfs diff=lfs merge=lfs -text
+DI-sheep/ui/public/demo.gif filter=lfs diff=lfs merge=lfs -text
+DI-sheep/ui/public/sound-disco.mp3 filter=lfs diff=lfs merge=lfs -text

DI-sheep/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2017 Google Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

DI-sheep/README.md

@@ -0,0 +1,103 @@
+# DI-sheep：深度强化学习 + 羊了个羊
+
+当 AI 技术的明珠——深度强化学习，遇到号称“通关率只有0.01%”的游戏“羊了个羊”，会碰撞出哪些奇思妙想呢？
+
+P.S. 路过记得点个 star ![stars - di-sheep](https://img.shields.io/github/stars/opendilab/di-sheep?style=social) ，持续更新ing。
+
+P.S.S. 想了解更多深度强化学习相关知识？快来 [DI-engine](https://github.com/opendilab/DI-engine) 训练自己的智能体。
+
+<div align="center">
+    <a href="https://github.com/opendilab/DI-sheep"><img width="500px" height="auto" src="https://github.com/opendilab/DI-sheep/blob/master/ui/public/demo.gif"></a>
+</div>
+
+# News
+- [bilibili] [羊群加不进去？那就来加猫群叭！深度强化学习版的羊了个羊来了](https://www.bilibili.com/video/BV1N24y1o7Lw/?spm_id_from=333.999.0.0)
+- [WeChat] [万事开头难？喵小DI玩“羊了个羊”可不难](https://mp.weixin.qq.com/s/4Z3WtkcWRp6x4x60RVELfQ)
+
+# 使用指南
+
+## 算法原理解析
+![disheep drawio](https://user-images.githubusercontent.com/33195032/191955286-7c309e9d-6e35-491f-93b3-b14cd1fe033f.png)
+
+## 快速上手
+
+- 如果想**在线试玩** --> [在线网页（改进中）](https://opendilab.net/sheep)
+- 如果想**本地部署/测试**
+  - 服务端（Python）
+    ```shell
+    # 预先安装好 Python3
+    cd service
+    pip install -r requirement.txt
+    FLASK_APP=app.py flask run  # 玩家试玩
+    # FLASK_APP=agent_app.py flask run  # 玩家 + AI 试玩
+    ```
+  - 客户端（react）
+    ```shell
+    # 预先安装好 node.js 和 react
+    cd ui
+    npm run build
+    npm run preview
+    ```
+    然后在网页中打开对应链接即可
+- 如果想进行完整的深度强化学习训练
+    ```shell
+    # 预先安装好 Python3
+    cd service
+    pip install -r requirement-train.txt
+    python3 -u sheep_ppo_main.py
+    ```
+- 如果想使用定义好的 gym 羊了个羊环境 --> 点个 star 之后直接暴力 CTRL C+V 拿走 `service/sheep_env.py` 尽情魔改
+- 如果想获得训练好的深度强化学习模型 --> 访问 [OpenDILab官网下载链接](https://opendilab.net/download/DI-sheep/) （目前提供了两种试玩模型，但智能体仍有很多进步空间）
+- 如果想了解更多深度强化学习相关知识 --> 欢迎参阅 [DI-engine](https://github.com/opendilab/DI-engine) 和[相关文档](https://di-engine-docs.readthedocs.io/zh_CN/latest/)
+- 如果想了解未来的更新计划 --> 请参阅[更新计划](#更新计划)
+- 如果有其他问题或想法 --> 欢迎 github ISSUE 区讨论，或是贡献 Pull requests
+
+## 项目结构
+```text
+.
+├── LICENSE
+├── ui                       --> react 网页前端
+└── service                  --> Python 核心模块（算法和服务端）
+    ├── app.py                  --> flask 服务 app (仅人类操作)
+    ├── agent_app.py                  --> flask 服务 app（人类+AI操作）
+    ├── requirement.txt         --> Python 依赖库列表
+    ├── sheep_env.py            --> gym 格式环境
+    ├── sheep_model.py          --> 基于 PyTorch 的 Actor-Critic 神经网络模型
+    ├── sheep_ppo_main.py       --> 基于 DI-engine 的深度强化学习训练主函数
+    ├── test_sheep_env.py       --> gym 格式环境的单元测试
+    └── test_sheep_model.py     --> 神经网络模型的单元测试
+```
+
+
+
+# 更新计划
+
+## 算法
+
+- [ ] 强化学习训练参数调整和算法微调
+- [ ] 提供可供本地试玩的模型权重
+- [ ] 详细的神经网络和强化学习算法设计文档
+- [ ] model-based RL 和 planning 算法
+- [ ] 神经网络压缩（用于部署） 
+
+## 环境
+- [ ] 添加原类型游戏（比如3tiles）中的各种道具
+- [ ] 牌的层数问题如何定义
+- [ ] 结合 JAX 优化环境运行速度
+- [ ] 更多结合 AI 的玩法设计
+
+
+## 应用
+- [x] 更多自定义主题和 BGM
+- [x] 在线网页端部署
+- [x] 样例神经网络权重下载
+- [ ] 更多 AI 训练时的行为分析
+- [ ] 移动端应用（欢迎Android/IOS开发者支持）
+
+
+# 致谢
+- react 前端部分主要参考 https://github.com/StreakingMan/solvable-sheep-game ，请大家也多多支持这个 repo 
+
+
+# License
+DI-sheep is released under the Apache 2.0 license.

DI-sheep/service/agent_app.py

@@ -0,0 +1,161 @@
+import time
+import numpy as np
+import torch
+from flask import Flask, request, jsonify, make_response
+from flask_restplus import Api, Resource, fields
+from threading import Thread
+from sheep_env import SheepEnv
+from sheep_model import SheepModel
+
+flask_app = Flask(__name__)
+app = Api(
+    app=flask_app,
+    version="0.0.1",
+    title="DI-sheep App",
+    description="Play Sheep with Deep Reinforcement Learning, Powered by OpenDILab"
+)
+
+name_space = app.namespace('DI-sheep', description='DI-sheep APIs')
+model = app.model(
+    'DI-sheep params', {
+        'command': fields.String(required=False, description="Command Field", help="reset, step"),
+        'argument': fields.Integer(required=False, description="Argument Field", help="reset->level, step->action"),
+    }
+)
+MAX_ENV_NUM = 50
+ENV_TIMEOUT_SECOND = 60
+envs = {}
+model = SheepModel(item_obs_size=80, item_num=30, global_obs_size=19)
+ckpt = torch.load('ckpt_best.pth.tar', map_location='cpu')['model']
+ckpt = {'item_encoder.encoder' + k.split('item_encoder')[-1] if 'item_encoder' in k else k: v for k, v in ckpt.items()}  # compatibility for v1 and v2 model
+model.load_state_dict(ckpt)
+
+
+def random_action(obs, env):
+    action_mask = obs['action_mask']
+    action = np.random.choice(len(action_mask), p=action_mask / action_mask.sum())
+    return action
+
+
+def env_monitor():
+    while True:
+        cur_time = time.time()
+        pop_keys = []
+        for k, v in envs.items():
+            if cur_time - v['update_time'] >= ENV_TIMEOUT_SECOND:
+                pop_keys.append(k)
+        for k in pop_keys:
+            envs.pop(k)
+        time.sleep(1)
+
+
+app.env_thread = Thread(target=env_monitor, daemon=True)
+app.env_thread.start()
+
+
+@name_space.route("/")
+class MainClass(Resource):
+
+    def options(self):
+        response = make_response()
+        response.headers.add("Access-Control-Allow-Origin", "*")
+        response.headers.add('Access-Control-Allow-Headers', "*")
+        response.headers.add('Access-Control-Allow-Methods', "*")
+        return response
+
+    @app.expect(model)
+    def post(self):
+        try:
+            t_start = time.time()
+            data = request.json
+            cmd, arg, uid = data['command'], data['argument'], data['uid']
+            ip = request.remote_addr + uid
+
+            if ip not in envs:
+                if cmd == 'reset':
+                    if len(envs) >= MAX_ENV_NUM:
+                        response = jsonify(
+                            {
+                                "statusCode": 501,
+                                "status": "No enough env resource, please wait a moment",
+                            }
+                        )
+                        response.headers.add('Access-Control-Allow-Origin', '*')
+                        return response
+                    else:
+                        env = SheepEnv(1, agent=True, max_padding=True)
+                        env.seed(0)
+                        envs[ip] = {'env': env, 'update_time': time.time()}
+                else:
+                    response = jsonify(
+                        {
+                            "statusCode": 501,
+                            "status": "No response for too long time, please reset the game",
+                        }
+                    )
+                    response.headers.add('Access-Control-Allow-Origin', '*')
+                    return response
+            else:
+                env = envs[ip]['env']
+                envs[ip]['update_time'] = time.time()
+
+            if cmd == 'reset':
+                obs = env.reset(arg)
+                action = model.compute_action(obs)
+                # action = random_action(obs, env)
+                scene = [item.to_json() for item in env.scene if item is not None]
+                response = jsonify(
+                    {
+                        "statusCode": 200,
+                        "status": "Execution action",
+                        "result": {
+                            "scene": scene,
+                            "max_item_num": env.total_item_num,
+                            "action": action,
+                        }
+                    }
+                )
+            elif cmd == 'step':
+                obs, _, done, _ = env.step(arg)
+                action = model.compute_action(obs)
+                # action = random_action(obs, env)
+                scene = [item.to_json() for item in env.scene if item is not None]
+                bucket = [item.to_json() for item in env.bucket]
+                response = jsonify(
+                    {
+                        "statusCode": 200,
+                        "status": "Execution action",
+                        "result": {
+                            "scene": scene,
+                            "bucket": bucket,
+                            "done": done,
+                            "action": action,
+                        }
+                    }
+                )
+            else:
+                response = jsonify({
+                    "statusCode": 500,
+                    "status": "Invalid command: {}".format(cmd),
+                })
+                response.headers.add('Access-Control-Allow-Origin', '*')
+                return response
+            print('backend process time: {}'.format(time.time() - t_start))
+            print('current env number: {}'.format(len(envs)))
+            response.headers.add('Access-Control-Allow-Origin', '*')
+            return response
+        except Exception as e:
+            import traceback
+            print(repr(e))
+            print(traceback.format_exc())
+            response = jsonify({
+                "statusCode": 500,
+                "status": "Could not execute action",
+            })
+            response.headers.add('Access-Control-Allow-Origin', '*')
+            return response
+
+if __name__ == "__main__":
+    flask_app.run(host='0.0.0.0')
+
+

DI-sheep/service/app.py

@@ -0,0 +1,142 @@
+import time
+from flask import Flask, request, jsonify, make_response
+from flask_restplus import Api, Resource, fields
+from threading import Thread
+from sheep_env import SheepEnv
+
+flask_app = Flask(__name__)
+app = Api(
+    app=flask_app,
+    version="0.0.1",
+    title="DI-sheep App",
+    description="Play Sheep with Deep Reinforcement Learning, Powered by OpenDILab"
+)
+
+name_space = app.namespace('DI-sheep', description='DI-sheep APIs')
+model = app.model(
+    'DI-sheep params', {
+        'command': fields.String(required=False, description="Command Field", help="reset, step"),
+        'argument': fields.Integer(required=False, description="Argument Field", help="reset->level, step->action"),
+    }
+)
+MAX_ENV_NUM = 50
+ENV_TIMEOUT_SECOND = 60
+envs = {}
+
+
+def env_monitor():
+    while True:
+        cur_time = time.time()
+        pop_keys = []
+        for k, v in envs.items():
+            if cur_time - v['update_time'] >= ENV_TIMEOUT_SECOND:
+                pop_keys.append(k)
+        for k in pop_keys:
+            envs.pop(k)
+        time.sleep(1)
+
+
+app.env_thread = Thread(target=env_monitor, daemon=True)
+app.env_thread.start()
+
+
+@name_space.route("/")
+class MainClass(Resource):
+
+    def options(self):
+        response = make_response()
+        response.headers.add("Access-Control-Allow-Origin", "*")
+        response.headers.add('Access-Control-Allow-Headers', "*")
+        response.headers.add('Access-Control-Allow-Methods', "*")
+        return response
+
+    @app.expect(model)
+    def post(self):
+        try:
+            t_start = time.time()
+            data = request.json
+            cmd, arg, uid = data['command'], data['argument'], data['uid']
+            ip = request.remote_addr
+            ip = str(ip) + str(uid)
+
+            if ip not in envs:
+                if cmd == 'reset':
+                    if len(envs) >= MAX_ENV_NUM:
+                        response = jsonify(
+                            {
+                                "statusCode": 501,
+                                "status": "No enough env resource, please wait a moment",
+                            }
+                        )
+                        response.headers.add('Access-Control-Allow-Origin', '*')
+                        return response
+                    else:
+                        env = SheepEnv(1, agent=False)
+                        envs[ip] = {'env': env, 'update_time': time.time()}
+                else:
+                    response = jsonify(
+                        {
+                            "statusCode": 501,
+                            "status": "No response for too long time, please reset the game",
+                        }
+                    )
+                    response.headers.add('Access-Control-Allow-Origin', '*')
+                    return response
+            else:
+                env = envs[ip]['env']
+                envs[ip]['update_time'] = time.time()
+            if cmd == 'reset':
+                env.reset(arg)
+                scene = [item.to_json() for item in env.scene if item is not None]
+                response = jsonify(
+                    {
+                        "statusCode": 200,
+                        "status": "Execution action",
+                        "result": {
+                            "scene": scene,
+                            "max_item_num": env.total_item_num,
+                        }
+                    }
+                )
+            elif cmd == 'step':
+                _, _, done, _ = env.step(arg)
+                scene = [item.to_json() for item in env.scene if item is not None]
+                bucket = [item.to_json() for item in env.bucket]
+                response = jsonify(
+                    {
+                        "statusCode": 200,
+                        "status": "Execution action",
+                        "result": {
+                            "scene": scene,
+                            "bucket": bucket,
+                            "done": done,
+                        }
+                    }
+                )
+            else:
+                response = jsonify({
+                    "statusCode": 500,
+                    "status": "Invalid command: {}".format(cmd),
+                })
+                response.headers.add('Access-Control-Allow-Origin', '*')
+                return response
+            print('backend process time: {}'.format(time.time() - t_start))
+            print('current env number: {}'.format(len(envs)))
+            response.headers.add('Access-Control-Allow-Origin', '*')
+            return response
+        except Exception as e:
+            import traceback
+            print(repr(e))
+            print(traceback.format_exc())
+            response = jsonify({
+                "statusCode": 500,
+                "status": "Could not execute action",
+            })
+            response.headers.add('Access-Control-Allow-Origin', '*')
+            return response
+
+if __name__ == "__main__":
+    flask_app.run(host='0.0.0.0')
+
+
+

DI-sheep/service/ckpt_best.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fda262054a2d69e44b9a2189c82be0203f412f432a37a7e9ec9b93dd0416e18
+size 3403809

DI-sheep/service/requirement-train.txt

@@ -0,0 +1 @@
+DI-engine>=0.4.5

DI-sheep/service/requirement.txt

@@ -0,0 +1,8 @@
+flask==1.1.2
+flask-restplus==0.13.0
+werkzeug==0.16.0
+jinja2==3.0.3
+markupsafe==2.0.1
+itsdangerous==2.0.1
+numpy
+gym

DI-sheep/service/sheep_env.py

@@ -0,0 +1,248 @@
+from typing import Tuple, Optional, Dict
+from collections import deque
+import copy
+import gym
+import json
+import uuid
+import numpy as np
+
+
+class Item:
+
+    def __init__(self, icon, offset, row, column):
+        self.icon = icon
+        self.offset = offset
+        self.row = row
+        self.column = column
+        self.uid = str(uuid.uuid4())
+        self.x = column * 100 + offset
+        self.y = row * 100 + offset
+        self.grid_x = self.x % 25
+        self.grid_y = self.y % 25
+        self.accessible = 1
+        self.visible = 1
+
+    def __repr__(self) -> str:
+        return 'icon({})'.format(self.icon)
+
+    def to_json(self):
+        return json.dumps(self, default=lambda o: o.__dict__ if hasattr(o, '__dict__') else int(o) if isinstance(o,np.int64) else "", sort_keys=True, indent=2)
+
+
+class SheepEnv(gym.Env):
+    max_level = 10
+    R = 10
+    icons = [i for i in range(10)]
+    ranges = [
+        [2, 6],
+        [1, 6],
+        [1, 7],
+        [0, 7],
+        [0, 8],
+    ]
+
+    def __init__(self, level: int, bucket_length: int = 7, agent: bool = True, max_padding: bool = False) -> None:
+        self.level = level
+        assert 1 <= self.level <= self.max_level
+        self.bucket_length = bucket_length
+        self.agent = agent
+        self.max_padding = max_padding
+        self._make_game()
+
+    def seed(self, seed: int) -> None:
+        self._seed = seed
+        np.random.seed(self._seed)
+
+    def _make_game(self) -> None:
+        # TODO wash scene
+        self.icon_pool = self.icons[:2 * self.level]
+        self.offset_pool = [0, 25, 50, 75][:1 + self.level]
+        self.selected_range = self.ranges[min(4, self.level - 1)]
+        self.max_item_per_icon = 9
+        self.max_level_item_num = len(self.icons) * self.max_item_per_icon + 2
+        if self.level >= 10:
+            self.item_per_icon = 6 + (self.level - 5 + 1) // 2
+            # add non-divisible items
+            self.item_non_div = 2
+        else:
+            self.item_per_icon = 6
+            self.item_non_div = 0
+        self.total_item_num = len(self.icon_pool) * self.item_per_icon + self.item_non_div
+
+        self.scene = []
+        self.bucket = deque(maxlen=self.bucket_length)
+
+        N = self.selected_range[1] - self.selected_range[0] - 1
+        for i in range(len(self.icon_pool)):
+            for j in range(self.item_per_icon):
+                row = self.selected_range[0] + np.int64(N * np.random.random())
+                column = self.selected_range[0] + np.int64(N * np.random.random())
+                offset = self.offset_pool[np.int64(np.random.random() * len(self.offset_pool))]
+                item = Item(self.icon_pool[i], offset, row, column)
+                self.scene.append(item)
+        if self.item_non_div > 0:
+            for icon in np.random.choice(self.icon_pool, size=self.item_non_div, replace=False):
+                row = self.selected_range[0] + np.int64(N * np.random.random())
+                column = self.selected_range[0] + np.int64(N * np.random.random())
+                offset = self.offset_pool[np.int64(np.random.random() * len(self.offset_pool))]
+                item = Item(int(icon), offset, row, column)
+                self.scene.append(item)
+
+        self.cur_item_num = len(self.scene)
+        self.reward_3tiles = self.R * 0.5 / (len(self.scene) // 3)
+
+        self._update_visible_accessible()
+        self._set_space()
+
+    def _update_visible_accessible(self) -> None:
+        for i in range(self.total_item_num):
+            covered_items = []
+            item1 = self.scene[i]
+            if item1 is None:
+                continue
+            item1.accessible = 1
+            for j in range(i + 1, self.total_item_num):
+                item2 = self.scene[j]
+                if item2 is None:
+                    continue
+                if not (item2.x + 100 <= item1.x or item2.x >= item1.x + 100 or item2.y + 100 <= item1.y
+                        or item2.y >= item1.y + 100):
+                    item1.accessible = 0
+                    if not self.agent:
+                        break
+                    covered_items.append(item2)
+            if self.agent and len(covered_items) > 0:
+                flag = np.zeros((2, 2)).astype(np.int64)  # core offset 50x50 is visible
+                core_x, core_y = item1.x + 25, item1.y + 25
+                for item in covered_items:
+                    min_x = max(core_x, item.x)
+                    max_x = min(core_x + 50, item.x + 100)
+                    min_y = max(core_y, item.y)
+                    max_y = min(core_y + 50, item.y + 100)
+                    if min_x < max_x or min_y < max_y:
+                        # top left: (max_x, max_y)
+                        # bottom right: (min_x, min_y)
+                        flag[(min_x - core_x) // 25:(max_x - core_x) // 25,
+                             (min_y - core_y) // 25:(max_y - core_y) // 25] = 1
+                        if flag.sum() == 4:
+                            break
+                item1.visible = int(flag.sum() < 4)
+            else:
+                item1.visible = 1
+
+    def _execute_action(self, action: int) -> float:
+        action_item = copy.deepcopy(self.scene[action])
+        assert action_item is not None, action
+        self.scene[action] = None
+        self.cur_item_num -= 1
+        same_items = []
+        for i in range(len(self.bucket)):
+            item = self.bucket[i]
+            if item.icon == action_item.icon:
+                same_items.append(item)
+
+        if len(same_items) == 2:
+            for item in same_items:
+                self.bucket.remove(item)
+            return copy.deepcopy(self.reward_3tiles)  # necessary deepcopy
+        else:
+            self.bucket.append(action_item)
+            return 0.
+
+    def reset(self, level: Optional[int] = None) -> Dict:
+        if level is not None:
+            self.level = level
+            assert 1 <= self.level <= self.max_level
+        self._make_game()
+        self._set_space()
+        return self._get_obs()
+
+    def close(self) -> None:
+        pass
+
+    # usually overwritten methods
+
+    def step(self, action: int) -> Tuple:
+        rew = self._execute_action(action)
+        self._update_visible_accessible()
+
+        obs = self._get_obs()
+        if self.cur_item_num == 0:
+            rew += self.R
+            done = True
+        elif len(self.bucket) == self.bucket_length:
+            rew -= self.R
+            done = True
+        else:
+            done = False
+        info = {}
+        return obs, rew, done, info
+
+    def _get_obs(self) -> Dict:
+        item_obs = np.zeros((self.total_item_num, self.item_size))
+        action_mask = np.zeros(self.total_item_num).astype(np.uint8)
+
+        L, N = self.L, self.N
+        p1, p2, p3 = L + N, L + N + N, L + N + N + 2
+        for i in range(len(self.scene)):
+            item = self.scene[i]
+            if item is None:
+                item_obs[i][L - 1] = 1
+            else:
+                item_obs[i][L + item.grid_x] = 1
+                item_obs[i][p1 + item.grid_y] = 1
+                item_obs[i][p3 + item.visible] = 1
+                if item.visible:
+                    item_obs[i][item.icon] = 1
+                    item_obs[i][p2 + item.accessible] = 1
+                else:
+                    item_obs[i][L - 2] = 1
+                if item.accessible:
+                    action_mask[i] = 1
+
+        bucket_obs = np.zeros(3 * len(self.icons))
+        bucket_icon_stat = [0 for _ in range(len(self.icons))]
+        for item in self.bucket:
+            bucket_icon_stat[item.icon] += 1
+        for i in range(len(bucket_icon_stat)):
+            bucket_obs[i * 3 + bucket_icon_stat[i]] = 1
+
+        global_obs = np.zeros(self.global_size)
+        if self.max_padding:
+            global_obs[self.cur_item_num // self.max_item_per_icon] = 1
+        else:
+            global_obs[self.cur_item_num // self.item_per_icon] = 1
+        global_obs[self.global_size - self.bucket_length - 1 + len(self.bucket)] = 1
+
+        return {
+            'item_obs': item_obs,
+            'bucket_obs': bucket_obs,
+            'global_obs': global_obs,
+            'action_mask': action_mask,
+        }
+
+    def _set_space(self) -> None:
+        if self.max_padding:
+            N = self.ranges[-1][1] - self.ranges[-1][0]
+            # icon + x + y + accessible + visible
+            L = len(self.icons) + 2  # +2 for not visible and move out
+        else:
+            N = self.selected_range[1] - self.selected_range[0]
+            # icon + x + y + accessible + visible
+            L = len(self.icon_pool) + 2  # +2 for not visible and move out
+        self.L, self.N = L, N
+        self.item_size = L + 4 * N * 2 + 2 + 2
+        if self.max_padding:
+            self.global_size = self.max_level_item_num // self.max_item_per_icon + 1 + self.bucket_length + 1
+        else:
+            self.global_size = self.total_item_num // self.item_per_icon + 1 + self.bucket_length + 1
+        self.observation_space = gym.spaces.Dict(
+            {
+                'item_obs': gym.spaces.Box(0, 1, dtype=np.float32, shape=(self.total_item_num, self.item_size)),
+                'bucket_obs': gym.spaces.Box(0, 1, dtype=np.float32, shape=(3 * len(self.icons), )),
+                'global_obs': gym.spaces.Box(0, 1, dtype=np.float32, shape=(self.global_size, )),
+                'action_mask': gym.spaces.Box(0, 1, dtype=np.float32, shape=(self.total_item_num, ))  # TODO
+            }
+        )
+        self.action_space = gym.spaces.Discrete(self.total_item_num)
+        self.reward_space = gym.spaces.Box(-self.R * 1.5, self.R * 1.5, dtype=np.float32)

DI-sheep/service/sheep_model.py

@@ -0,0 +1,126 @@
+import torch
+import torch.nn as nn
+import treetensor.torch as ttorch
+from ding.torch_utils import Transformer, MLP, unsqueeze, to_tensor
+
+
+class ItemEncoder(nn.Module):
+    encoder_type = ['TF', 'MLP', 'two_stage_MLP']
+
+    def __init__(self, item_obs_size=60, item_num=30, item_encoder_type='TF', hidden_size=64, activation=nn.ReLU()):
+        super(ItemEncoder, self).__init__()
+        assert item_encoder_type in self.encoder_type, "not support item encoder type: {}/{}".format(item_encoder_type, self.encoder_type)
+        self.item_encoder_type = item_encoder_type
+        self.item_num = item_num
+        self.hidden_size = hidden_size
+
+        if self.item_encoder_type == 'TF':
+            self.encoder = Transformer(
+                item_obs_size,
+                hidden_dim=2 * hidden_size,
+                output_dim=hidden_size,
+                activation=activation
+            )
+        elif self.item_encoder_type == 'MLP':
+            self.encoder = MLP(
+                item_obs_size,
+                hidden_size,
+                hidden_size,
+                layer_num=3,
+                activation=activation
+            )
+        elif self.item_encoder_type == 'two_stage_MLP':
+            self.trans_len = 16
+            self.encoder_1 = MLP(
+                item_obs_size,
+                hidden_size,
+                self.trans_len,
+                layer_num=3,
+                activation=activation
+            )
+            self.encoder_2 = MLP(
+                self.trans_len*self.item_num,
+                hidden_size,
+                self.item_num*hidden_size,
+                layer_num=2,
+                activation=activation
+            )
+
+    def forward(self, item_obs):
+        if self.item_encoder_type == 'two_stage_MLP':
+            item_embedding_1 = self.encoder_1(item_obs)   # (B, M, L)
+            item_embedding_2 = torch.reshape(item_embedding_1, [-1, self.trans_len*self.item_num])
+            item_embedding = self.encoder_2(item_embedding_2)
+            item_embedding = torch.reshape(item_embedding, [-1, self.item_num, self.hidden_size])
+        else:
+            item_embedding = self.encoder(item_obs)
+        return item_embedding
+
+
+class SheepModel(nn.Module):
+    mode = ['compute_actor', 'compute_critic', 'compute_actor_critic']
+
+    def __init__(self, item_obs_size=60, item_num=30, item_encoder_type='TF', bucket_obs_size=30, global_obs_size=17, hidden_size=64, activation=nn.ReLU(), ttorch_return=False):
+        super(SheepModel, self).__init__()
+        self.item_encoder = ItemEncoder(item_obs_size, item_num, item_encoder_type, hidden_size, activation=activation)
+        self.bucket_encoder = MLP(bucket_obs_size, hidden_size, hidden_size, layer_num=3, activation=activation)
+        self.global_encoder = MLP(global_obs_size, hidden_size, hidden_size, layer_num=2, activation=activation)
+        self.value_head = nn.Sequential(
+            MLP(hidden_size, hidden_size, hidden_size, layer_num=2, activation=activation), nn.Linear(hidden_size, 1)
+        )
+        self.ttorch_return = ttorch_return
+
+    def compute_actor(self, x):
+        item_embedding = self.item_encoder(x['item_obs'])
+        bucket_embedding = self.bucket_encoder(x['bucket_obs'])
+        global_embedding = self.global_encoder(x['global_obs'])
+
+        key = item_embedding
+        query = bucket_embedding + global_embedding
+        query = query.unsqueeze(1)
+        logit = (key * query).sum(2)
+        logit.masked_fill_(~x['action_mask'].bool(), value=-1e9)
+        if self.ttorch_return:
+            return logit
+        else:
+            return {'logit': logit}
+
+    def compute_critic(self, x):
+        item_embedding = self.item_encoder(x['item_obs'])
+        bucket_embedding = self.bucket_encoder(x['bucket_obs'])
+        global_embedding = self.global_encoder(x['global_obs'])
+
+        embedding = item_embedding.mean(1) + bucket_embedding + global_embedding
+        value = self.value_head(embedding)
+        if self.ttorch_return:
+            return value.squeeze(1)
+        else:
+            return {'value': value.squeeze(1)}
+
+    def compute_actor_critic(self, x):
+        item_embedding = self.item_encoder(x['item_obs'])
+        bucket_embedding = self.bucket_encoder(x['bucket_obs'])
+        global_embedding = self.global_encoder(x['global_obs'])
+
+        key = item_embedding
+        query = bucket_embedding + global_embedding
+        query = query.unsqueeze(1)
+        logit = (key * query).sum(2)
+        logit.masked_fill_(~x['action_mask'].bool(), value=-1e9)
+
+        embedding = item_embedding.mean(1) + bucket_embedding + global_embedding
+        value = self.value_head(embedding)
+        if self.ttorch_return:
+            return ttorch.as_tensor({'logit': logit, 'value': value.squeeze(1)})
+        else:
+            return {'logit': logit, 'value': value.squeeze(1)}
+
+    def forward(self, x, mode):
+        assert mode in self.mode, "not support forward mode: {}/{}".format(mode, self.mode)
+        return getattr(self, mode)(x)
+
+    def compute_action(self, x):
+        x = unsqueeze(to_tensor(x))
+        with torch.no_grad():
+            logit = self.compute_actor(x)['logit']
+            return logit.argmax(dim=-1)[0].item()

DI-sheep/service/sheep_ppo_level9_main.py

@@ -0,0 +1,123 @@
+import os
+from easydict import EasyDict
+from tensorboardX import SummaryWriter
+from ding.config import compile_config
+from ding.envs import create_env_manager, DingEnvWrapper, EvalEpisodeReturnEnv
+from ding.policy import PPOPolicy
+from ding.worker import BaseLearner, create_serial_collector, InteractionSerialEvaluator
+from ding.utils import set_pkg_seed
+
+from sheep_env import SheepEnv
+from sheep_model import SheepModel
+
+sheep_ppo_config = dict(
+    exp_name='level9/sheep_ppo_MLP_1M_seed0',
+    env=dict(
+        env_id='Sheep-v0',
+        level=10,
+        collector_env_num=8,
+        evaluator_env_num=10,
+        n_evaluator_episode=10,
+        # stop_value=15,
+        stop_value=1e6,     # to run fixed env step
+    ),
+    policy=dict(
+        cuda=True,
+        recompute_adv=True,
+        action_space='discrete',
+        model=dict(),
+        learn=dict(
+            epoch_per_collect=10,
+            batch_size=320,
+            learning_rate=3e-4,
+            value_weight=0.5,
+            entropy_weight=0.001,
+            clip_ratio=0.2,
+            adv_norm=False,
+            value_norm=True,
+        ),
+        collect=dict(
+            n_sample=3200,
+            discount_factor=0.99,
+            gae_lambda=0.95,
+        ),
+        eval=dict(evaluator=dict(eval_freq=500, )),
+    ),
+)
+sheep_ppo_config = EasyDict(sheep_ppo_config)
+main_config = sheep_ppo_config
+
+sheep_ppo_create_config = dict(
+    env=dict(
+        type='mujoco',
+        import_names=['dizoo.mujoco.envs.mujoco_env'],
+    ),
+    env_manager=dict(type='subprocess'),
+    policy=dict(type='ppo', ),
+)
+sheep_ppo_create_config = EasyDict(sheep_ppo_create_config)
+create_config = sheep_ppo_create_config
+
+
+def sheep_env_fn(level):
+    return DingEnvWrapper(
+        SheepEnv(level), cfg={'env_wrapper': [
+            lambda env: EvalEpisodeReturnEnv(env),
+        ]}
+    )
+
+
+def main(input_cfg, seed, max_env_step=int(1e6), max_train_iter=int(1e6)):
+    cfg, create_cfg = input_cfg
+    cfg = compile_config(cfg, seed=seed, auto=True, create_cfg=create_cfg)
+    collector_env = create_env_manager(
+        cfg.env.manager, [lambda: sheep_env_fn(cfg.env.level) for _ in range(cfg.env.collector_env_num)]
+    )
+    evaluator_env = create_env_manager(
+        cfg.env.manager, [lambda: sheep_env_fn(cfg.env.level) for _ in range(cfg.env.evaluator_env_num)]
+    )
+    collector_env.seed(cfg.seed, dynamic_seed=False)
+    evaluator_env.seed(cfg.seed, dynamic_seed=False)
+    set_pkg_seed(cfg.seed, use_cuda=cfg.policy.cuda)
+
+    obs_space = collector_env._env_ref.observation_space
+    model = SheepModel(
+        obs_space['item_obs'].shape[1],
+        obs_space['item_obs'].shape[0],
+        'TF',
+        obs_space['bucket_obs'].shape[0],
+        obs_space['global_obs'].shape[0]
+    )
+    policy = PPOPolicy(cfg.policy, model=model, enable_field=['learn', 'collect', 'eval'])
+
+    tb_logger = SummaryWriter(os.path.join('./{}/log/'.format(cfg.exp_name), 'serial'))
+    learner = BaseLearner(cfg.policy.learn.learner, policy.learn_mode, tb_logger, exp_name=cfg.exp_name)
+    collector = create_serial_collector(
+        cfg.policy.collect.collector,
+        env=collector_env,
+        policy=policy.collect_mode,
+        tb_logger=tb_logger,
+        exp_name=cfg.exp_name
+    )
+    evaluator = InteractionSerialEvaluator(
+        cfg.policy.eval.evaluator, evaluator_env, policy.eval_mode, tb_logger, exp_name=cfg.exp_name
+    )
+
+    learner.call_hook('before_run')
+
+    while True:
+        if evaluator.should_eval(learner.train_iter):
+            stop, reward = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
+            if stop:
+                break
+        new_data = collector.collect(train_iter=learner.train_iter)
+
+        learner.train(new_data, collector.envstep)
+        if collector.envstep >= max_env_step or learner.train_iter >= max_train_iter:
+            break
+
+    learner.call_hook('after_run')
+
+
+if __name__ == "__main__":
+    main([main_config, create_config], seed=0)

DI-sheep/service/sheep_ppo_main.py

@@ -0,0 +1,122 @@
+import os
+from easydict import EasyDict
+from tensorboardX import SummaryWriter
+from ding.config import compile_config
+from ding.envs import create_env_manager, DingEnvWrapper, EvalEpisodeReturnEnv
+from ding.policy import PPOPolicy
+from ding.worker import BaseLearner, create_serial_collector, InteractionSerialEvaluator
+from ding.utils import set_pkg_seed
+
+from sheep_env import SheepEnv
+from sheep_model import SheepModel
+
+sheep_ppo_config = dict(
+    exp_name='sheep_ppo_seed0',
+    env=dict(
+        env_id='Sheep-v0',
+        level=2,
+        collector_env_num=8,
+        evaluator_env_num=10,
+        n_evaluator_episode=10,
+        stop_value=15,
+    ),
+    policy=dict(
+        cuda=True,
+        recompute_adv=True,
+        action_space='discrete',
+        model=dict(),
+        learn=dict(
+            epoch_per_collect=10,
+            batch_size=320,
+            learning_rate=3e-4,
+            value_weight=0.5,
+            entropy_weight=0.001,
+            clip_ratio=0.2,
+            adv_norm=False,
+            value_norm=True,
+        ),
+        collect=dict(
+            n_sample=3200,
+            discount_factor=0.99,
+            gae_lambda=0.95,
+        ),
+        eval=dict(evaluator=dict(eval_freq=500, )),
+    ),
+)
+sheep_ppo_config = EasyDict(sheep_ppo_config)
+main_config = sheep_ppo_config
+
+sheep_ppo_create_config = dict(
+    env=dict(
+        type='mujoco',
+        import_names=['dizoo.mujoco.envs.mujoco_env'],
+    ),
+    env_manager=dict(type='subprocess'),
+    policy=dict(type='ppo', ),
+)
+sheep_ppo_create_config = EasyDict(sheep_ppo_create_config)
+create_config = sheep_ppo_create_config
+
+
+def sheep_env_fn(level):
+    return DingEnvWrapper(
+        SheepEnv(level), cfg={'env_wrapper': [
+            lambda env: EvalEpisodeReturnEnv(env),
+        ]}
+    )
+
+
+def main(input_cfg, seed, max_env_step=int(1e7), max_train_iter=int(1e7)):
+    cfg, create_cfg = input_cfg
+    cfg = compile_config(cfg, seed=seed, auto=True, create_cfg=create_cfg)
+    collector_env = create_env_manager(
+        cfg.env.manager, [lambda: sheep_env_fn(cfg.env.level) for _ in range(cfg.env.collector_env_num)]
+    )
+    evaluator_env = create_env_manager(
+        cfg.env.manager, [lambda: sheep_env_fn(cfg.env.level) for _ in range(cfg.env.evaluator_env_num)]
+    )
+    collector_env.seed(cfg.seed, dynamic_seed=False)
+    evaluator_env.seed(cfg.seed, dynamic_seed=False)
+    set_pkg_seed(cfg.seed, use_cuda=cfg.policy.cuda)
+
+    obs_space = collector_env._env_ref.observation_space
+    model = SheepModel(
+        obs_space['item_obs'].shape[1],
+        obs_space['item_obs'].shape[0],
+        'TF',
+        obs_space['bucket_obs'].shape[0],
+        obs_space['global_obs'].shape[0]
+    )
+    policy = PPOPolicy(cfg.policy, model=model, enable_field=['learn', 'collect', 'eval'])
+
+    tb_logger = SummaryWriter(os.path.join('./{}/log/'.format(cfg.exp_name), 'serial'))
+    learner = BaseLearner(cfg.policy.learn.learner, policy.learn_mode, tb_logger, exp_name=cfg.exp_name)
+    collector = create_serial_collector(
+        cfg.policy.collect.collector,
+        env=collector_env,
+        policy=policy.collect_mode,
+        tb_logger=tb_logger,
+        exp_name=cfg.exp_name
+    )
+    evaluator = InteractionSerialEvaluator(
+        cfg.policy.eval.evaluator, evaluator_env, policy.eval_mode, tb_logger, exp_name=cfg.exp_name
+    )
+
+    learner.call_hook('before_run')
+
+    while True:
+        if evaluator.should_eval(learner.train_iter):
+            stop, reward = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
+            if stop:
+                break
+        new_data = collector.collect(train_iter=learner.train_iter)
+
+        learner.train(new_data, collector.envstep)
+        if collector.envstep >= max_env_step or learner.train_iter >= max_train_iter:
+            break
+
+    learner.call_hook('after_run')
+
+
+if __name__ == "__main__":
+    main([main_config, create_config], seed=0)

DI-sheep/service/test_sheep_env.py

@@ -0,0 +1,23 @@
+import pytest
+import numpy as np
+from sheep_env import SheepEnv
+
+
+@pytest.mark.unittest
+def test_naive():
+    env = SheepEnv(level=9)
+    obs = env.reset()
+    print(env.observation_space)
+
+    while True:
+        action_mask = obs['action_mask']
+        action = np.random.choice(len(action_mask), p=action_mask / action_mask.sum())
+        obs, rew, done, info = env.step(action)
+        print(env.bucket, rew, done)
+        assert isinstance(obs, dict)
+        assert set(obs.keys()) == set(['item_obs', 'bucket_obs', 'global_obs', 'action_mask'])
+        assert np.isscalar(rew)
+        assert isinstance(done, bool)
+        assert isinstance(info, dict)
+        if done:
+            break

DI-sheep/service/test_sheep_model.py

@@ -0,0 +1,19 @@
+import pytest
+import torch
+from sheep_model import SheepModel
+
+
+@pytest.mark.unittest
+def test_naive():
+    B, M = 3, 30
+    model = SheepModel(item_num=30, item_encoder_type='two_stage_MLP')
+    data = {
+        'item_obs': torch.randn(B, M, 60),
+        'bucket_obs': torch.randn(B, 30),
+        'global_obs': torch.randn(B, 17),
+        'action_mask': torch.randint(0, 2, size=(M, )).bool()
+    }
+
+    output = model.forward(data, mode='compute_actor_critic')
+    assert output['logit'].shape == (B, M)
+    assert output['value'].shape == (B, )

DI-sheep/ui/.commitlintrc.cjs

@@ -0,0 +1,9 @@
+/**
+ * commitlint config
+ * @ref http://commitlint.js.org/
+ * @desc generated at 9/15/2022, 12:52:11 PM by [email protected]
+ */
+
+module.exports = {
+    extends: ['@commitlint/config-conventional'],
+};

DI-sheep/ui/.editorconfig

@@ -0,0 +1,15 @@
+# https://editorconfig.org/
+# generated at 9/15/2022, 12:52:15 PM by [email protected]
+# 该配置文件是给webstorm使用的
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false

DI-sheep/ui/.eslintignore

@@ -0,0 +1,9 @@
+# generated at 9/15/2022, 12:51:49 PM by [email protected]
+
+node_modules
+dist
+
+!.eslintrc.js
+!.prettierrc.js
+!.commitlintrc.js
+!.lintstagedrc.js

DI-sheep/ui/.eslintrc.cjs

@@ -0,0 +1,28 @@
+/**
+ * eslint config
+ * @ref http://eslint.cn/
+ * @desc generated at 9/15/2022, 12:51:49 PM by [email protected]
+ */
+
+module.exports = {
+    root: true,
+    parser: '@typescript-eslint/parser',
+    parserOptions: {
+        sourceType: 'module',
+        ecmaVersion: 11,
+    },
+    env: {
+        es6: true,
+        node: true,
+    },
+    plugins: ['eslint-plugin-prettier', '@typescript-eslint'],
+    extends: [
+        'eslint:recommended',
+        'plugin:react/recommended',
+        'plugin:@typescript-eslint/recommended',
+        'plugin:prettier/recommended',
+    ],
+    rules: {
+        'prettier/prettier': 'error',
+    },
+};

DI-sheep/ui/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+# dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

DI-sheep/ui/.husky/commit-msg

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx --no-install commitlint --edit 

DI-sheep/ui/.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx lint-staged

DI-sheep/ui/.lintstagedrc.cjs

@@ -0,0 +1,11 @@
+/**
+ * lint-staged config
+ * @ref https://www.npmjs.com/package/lint-staged
+ * @desc generated at 9/15/2022, 12:52:11 PM by [email protected]
+ */
+
+module.exports = {
+    '!(dist/**/*)*.{[tj]s,[tj]sx,[cm]js}': ['eslint --fix'],
+    '!(dist/**/*)*.json': ['prettier --write'],
+    '!(dist/**/*)*.{css,scss}': ['stylelint --fix'],
+};

DI-sheep/ui/.prettierrc.cjs

@@ -0,0 +1,12 @@
+/**
+ * prettier config
+ * @ref https://prettier.io/
+ * @desc generated at 9/15/2022, 12:51:48 PM by [email protected]
+ */
+
+module.exports = {
+    tabWidth: 4,
+    singleQuote: true,
+    htmlWhitespaceSensitivity: 'css',
+    endOfLine: 'lf'
+}

DI-sheep/ui/.stylelintrc.cjs

@@ -0,0 +1,12 @@
+/**
+ * stylelint config
+ * @ref https://stylelint.io/
+ * @desc generated at 9/15/2022, 12:51:58 PM by [email protected]
+ */
+
+module.exports = {
+    extends: [
+        'stylelint-config-standard-scss',
+        'stylelint-config-prettier-scss',
+    ],
+};

DI-sheep/ui/.versionrc.cjs

@@ -0,0 +1,5 @@
+module.exports = {
+    scripts: {
+        postbump: 'yarn build && git add dist/*',
+    },
+};

DI-sheep/ui/LICENSE.md

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

DI-sheep/ui/README.md

@@ -0,0 +1,3 @@
+# react 网页前端
+
+主要参考 https://github.com/StreakingMan/solvable-sheep-game ，请大家也多多支持这个 repo 

DI-sheep/ui/index.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <link rel="icon" type="image/svg+xml" href="/favicon.ico" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <meta
+            name="description"
+            content="DI-sheep"
+        />
+        <title>DI-sheep</title>
+        <script>
+            var _hmt = _hmt || [];
+            (function () {
+                var hm = document.createElement('script');
+                hm.src =
+                    'https://hm.baidu.com/hm.js?0a5701b62c5da53aa10f9096fccb377c';
+                var s = document.getElementsByTagName('script')[0];
+                s.parentNode.insertBefore(hm, s);
+            })();
+        </script>
+    </head>
+    <body>
+        <div id="root"></div>
+        <script type="module" src="/src/main.tsx"></script>
+        <script
+            async
+            src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"
+        ></script>
+    </body>
+</html>

DI-sheep/ui/package.json

@@ -0,0 +1,41 @@
+{
+    "name": "DI-sheep-ui",
+    "private": false,
+    "version": "0.0.1",
+    "type": "module",
+    "scripts": {
+        "dev": "vite",
+        "build": "tsc && vite build",
+        "preview": "vite preview",
+        "prepare": "husky install",
+        "release:first": "standard-version -- --first-release",
+        "release": "standard-version --commit-all"
+    },
+    "dependencies": {
+        "react": "^18.2.0",
+        "react-dom": "^18.2.0"
+    },
+    "devDependencies": {
+        "@commitlint/cli": "^17.1.2",
+        "@commitlint/config-conventional": "^17.1.0",
+        "@types/react": "^18.0.20",
+        "@types/react-dom": "^18.0.6",
+        "@typescript-eslint/eslint-plugin": "^5.37.0",
+        "@typescript-eslint/parser": "^5.37.0",
+        "@vitejs/plugin-react": "^2.1.0",
+        "eslint": "^8.23.1",
+        "eslint-config-prettier": "^8.5.0",
+        "eslint-plugin-prettier": "^4.2.1",
+        "eslint-plugin-react": "^7.31.8",
+        "husky": "^8.0.1",
+        "lint-staged": "^13.0.3",
+        "prettier": "^2.7.1",
+        "standard-version": "^9.5.0",
+        "stylelint": "^14.11.0",
+        "stylelint-config-prettier-scss": "^0.0.1",
+        "stylelint-config-standard-scss": "^5.0.0",
+        "typescript": "^4.6.4",
+        "vite": "^3.1.0"
+    },
+    "license": "SEE LICENSE IN LICENSE.md"
+}

DI-sheep/ui/public/sound-disco.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b6f7960a64624890e0c59a9e8a8fb02a95c6449091b98d1fa4817c288f638c
+size 8299694

DI-sheep/ui/src/App.css

@@ -0,0 +1,110 @@
+#root {
+    text-align: center;
+    width: 100%;
+    max-width: 500px;
+    margin: 0 auto;
+}
+
+.app {
+    width: 100%;
+    margin: 0 auto;
+}
+
+.scene-container {
+    width: 100%;
+    padding-bottom: 100%;
+    position: relative;
+    margin: 10% 0;
+}
+
+.scene-inner {
+    position: absolute;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    top: 0;
+    overflow: visible;
+    font-size: 28px;
+}
+
+.symbol {
+    width: 12.5%;
+    padding-bottom: 12.5%;
+    position: absolute;
+    transition: 150ms;
+    left: 0;
+    top: 0;
+    border-radius: 8px;
+}
+
+.symbol-inner {
+    position: absolute;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    top: 0;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    border-radius: 8px;
+    border: 2px solid #444;
+    transition: 0.3s;
+    overflow: hidden;
+    user-select: none;
+}
+
+.symbol-inner img {
+    width: 100%;
+    height: 100%;
+    object-fit: contain;
+}
+
+.queue-container {
+    border-radius: 8px;
+    width: 100%;
+    padding-bottom: 15%;
+    border: 2px solid gray;
+    margin-bottom: 16px;
+}
+
+.flex-container {
+    display: flex;
+    gap: 8px;
+}
+
+.flex-center {
+    justify-content: center;
+    align-items: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}
+
+.flex-between {
+    justify-content: space-between;
+    align-items: center;
+}
+
+.modal {
+    position: fixed;
+    width: 100vw;
+    height: 100vh;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    backdrop-filter: blur(10px);
+    background-color: rgb(255 255 255 / 40%);
+    top: 0;
+    left: 0;
+}
+
+.bgm-button {
+    position: fixed;
+    left: 0;
+    top: 0;
+    padding: 4px;
+    width: 36px;
+    height: 36px;
+}

DI-sheep/ui/src/App.tsx

@@ -0,0 +1,449 @@
+import React, {
+    FC,
+    MouseEventHandler,
+    useEffect,
+    useRef,
+    useState,
+} from 'react';
+
+import './App.css';
+import { GithubIcon } from './GithubIcon';
+import { randomString, waitTimeout } from './utils';
+import { defaultTheme } from './themes/default';
+import { Icon, Theme } from './themes/interface';
+import { fishermanTheme } from './themes/fisherman';
+import { diTheme } from './themes/di';
+import { mhlTheme } from './themes/mhl';
+import { yhdTheme } from './themes/yhd';
+
+const themes = [defaultTheme, fishermanTheme, diTheme, mhlTheme, yhdTheme];
+
+const maxLevel = 10;
+const target_url = 'http://localhost:5002/DI-sheep/';
+
+interface MySymbol {
+    id: string;
+    status: number; // 0->1->2
+    isCover: boolean;
+    isAgentTarget: boolean;
+    x: number;
+    y: number;
+    icon: Icon;
+}
+
+type Scene = MySymbol[];
+
+// 8*8 grid with factor 4 (32x32)
+const makeScene: (level: number, icons: Icon[], new_scene_data: string[], agent_action: number) => Scene = (level, icons, new_scene_data, agent_action) => {
+    const curLevel = Math.min(maxLevel, level);
+    const iconPool = icons.slice(0, 2 * curLevel);
+
+    const scene: Scene = [];
+
+    for (const raw_data of new_scene_data) {
+        const data = JSON.parse(raw_data);
+
+        const count = scene.length;
+        scene.push({
+            isCover: !data.accessible,
+            // isCover: !data.visible,  // for viz debug
+            isAgentTarget: count === agent_action,
+            status: 0,
+            icon: iconPool[data.icon],
+            id: data.uid,
+            x: data.x,
+            y: data.y,
+        });
+    }
+    return scene;
+};
+
+
+interface SymbolProps extends MySymbol {
+    onClick: MouseEventHandler;
+}
+
+const Symbol: FC<SymbolProps> = ({ x, y, icon, isCover, isAgentTarget, status, onClick }) => {
+    return (
+        <div
+            className="symbol"
+            style={{
+                transform: `translateX(${x}%) translateY(${y}%)`,
+                backgroundColor: isAgentTarget ? 'orange' : isCover ? '#999' : 'white',
+                opacity: status < 2 ? 1 : 0,
+            }}
+            onClick={onClick}
+        >
+            <div
+                className="symbol-inner"
+                style={{ opacity: isCover ? 0.5 : 1 }}
+            >
+                {typeof icon.content === 'string' ? (
+                    <i>{icon.content}</i>
+                ) : (
+                    icon.content
+                )}
+            </div>
+        </div>
+    );
+};
+
+const uidApp = 'uid' + crypto.randomUUID();
+const App: FC = () => {
+    const [curTheme, setCurTheme] = useState<Theme<any>>(diTheme);
+    const [level, setLevel] = useState<number>(1);
+    const [maxItemNum, setMaxItemNum] = useState<number>(0);
+    const [resItemNum, setResItemNum] = useState<number>(0);
+    
+    const [startTime, setStartTime] = useState<number>(0);
+    const [now, setNow] = useState<number>(0);
+    const [usedTime, setUsedTime] = useState<number>(0);
+    const intervalRef = useRef<NodeJS.Timeout | null>(null);
+    
+    const [queue, setQueue] = useState<MySymbol[]>([]);
+    const [sortedQueue, setSortedQueue] = useState<
+        Record<MySymbol['id'], number>
+    >({});
+    const [finished, setFinished] = useState<boolean>(false);
+    const [tipText, setTipText] = useState<string>('');
+    const [animating, setAnimating] = useState<boolean>(false);
+    const [expired, setExpired] = useState<boolean>(false);
+    const [lastAgentTarget, setLastAgentTarget] = useState<number>(-1);
+    const [scene, setScene] = useState<Scene>(makeScene(level, curTheme.icons, [], -1));  // placeholder
+
+    // audio
+    const soundRefMap = useRef<Record<string, HTMLAudioElement>>({});
+
+    const bgmRef = useRef<HTMLAudioElement>(null);
+    const [bgmOn, setBgmOn] = useState<boolean>(false);
+    const [once, setOnce] = useState<boolean>(false);
+    useEffect(() => {
+        if (!bgmRef.current) return;
+        if (bgmOn) {
+            bgmRef.current.volume = 0.5;
+            bgmRef.current.play();
+        } else {
+            bgmRef.current?.pause();
+        }
+    }, [bgmOn]);
+
+    // change themes
+    useEffect(() => {
+        restart(level);
+    }, [curTheme]);
+
+    useEffect(() => {
+        if (finished || queue.length >= 7 || expired) return;
+        if (startTime && now) setUsedTime(now - startTime);
+    }, [now]);
+    const startTimer = () => {
+        setStartTime(Date.now() - 0);
+        setNow(Date.now());
+        intervalRef.current && clearInterval(intervalRef.current);
+        intervalRef.current = setInterval(() => {
+            setNow(Date.now());
+        }, 10);
+    };
+
+    // sort queue
+    useEffect(() => {
+        const cache: Record<string, MySymbol[]> = {};
+        for (const symbol of queue) {
+            if (cache[symbol.icon.name]) {
+                cache[symbol.icon.name].push(symbol);
+            } else {
+                cache[symbol.icon.name] = [symbol];
+            }
+        }
+        const temp = [];
+        for (const symbols of Object.values(cache)) {
+            temp.push(...symbols);
+        }
+        const updateSortedQueue: typeof sortedQueue = {};
+        let x = 50;
+        for (const symbol of temp) {
+            updateSortedQueue[symbol.id] = x;
+            x += 100;
+        }
+        setSortedQueue(updateSortedQueue);
+    }, [queue]);
+
+    const update = (new_scene_data: string[]) => {
+        for (const raw_data of new_scene_data) {
+            const data = JSON.parse(raw_data);
+            const find = scene.find((s) => s.id === data.uid);
+            if (find) {
+                if (find.status === 0) {
+                    find.isCover = !data.accessible;
+                }
+            }
+        }
+        if (scene.length > 0) {
+            scene.slice()[lastAgentTarget].isAgentTarget = false;
+        }
+    };
+
+    // TODO
+    const pop = () => {
+        if (!queue.length) return;
+        const updateQueue = queue.slice();
+        const symbol = updateQueue.shift();
+        if (!symbol) return;
+        const find = scene.find((s) => s.id === symbol.id);
+        if (find) {
+            setQueue(updateQueue);
+            find.status = 0;
+            find.x = 100 * Math.floor(8 * Math.random());
+            find.y = 700;
+        }
+    };
+
+    const levelUp = () => {
+        if (level >= maxLevel) {
+            return;
+        }
+        setFinished(false);
+        setTipText('');
+        setLevel(level + 1);
+        setQueue([]);
+        fetch(target_url,
+          {
+            headers: {
+              'Accept': 'application/json',
+              'Content-Type': 'application/json'
+            },
+            method: 'POST',
+            body: JSON.stringify({command: 'reset', argument: level + 1, uid: uidApp})
+          })
+          .then(response => response.json())
+          .then(response => {
+            setScene(makeScene(level + 1, curTheme.icons, response.result.scene, response.result.action));
+            setLastAgentTarget(response.result.action);
+            setMaxItemNum(response.result.max_item_num);
+            setResItemNum(response.result.max_item_num);
+            setUsedTime(0);
+            startTimer();
+        });
+    };
+
+    const restart = (level: number) => {
+        setFinished(false);
+        setExpired(false);
+        setTipText('');
+        setUsedTime(0);  // necessary 
+        setQueue([]);
+        fetch(target_url,
+          {
+            headers: {
+              'Accept': 'application/json',
+              'Content-Type': 'application/json'
+            },
+            method: 'POST',
+            body: JSON.stringify({command: 'reset', argument: level, uid: uidApp})
+          })
+          .then(response => response.json())
+          .then(response => {
+            setScene(makeScene(level, curTheme.icons, response.result.scene, response.result.action));
+            setLastAgentTarget(response.result.action);
+            setMaxItemNum(response.result.max_item_num);
+            setResItemNum(response.result.max_item_num);
+            setUsedTime(0);
+            startTimer();
+        });
+    };
+
+    const clickSymbol = async (idx: number) => {
+        // console.time("process");
+        if (idx === -1) {
+            idx = lastAgentTarget;
+        }
+        if (finished || animating) return;
+        if (queue.length >= 7) {
+            setTipText('挑战失败');
+            setFinished(true);
+            return;
+        }
+
+        if (!once) {
+            setBgmOn(true);
+            setOnce(true);
+            setUsedTime(0);
+            startTimer();
+        }
+
+        const updateScene = scene.slice();
+        const symbol = updateScene[idx];
+        if (symbol.isCover || symbol.status !== 0) return;
+        symbol.status = 1;
+
+        if (soundRefMap.current) {
+            soundRefMap.current[symbol.icon.clickSound].currentTime = 0;
+            soundRefMap.current[symbol.icon.clickSound].play();
+        }
+        let status = false;
+
+        fetch(target_url,
+          {
+            headers: {
+              'Accept': 'application/json',
+              'Content-Type': 'application/json'
+            },
+            method: 'POST',
+            body: JSON.stringify({command: 'step', argument: idx, uid: uidApp})
+          })
+          .then(response => response.json())
+          .then(response => {
+            setFinished(response.statusCode === 501)
+            if (response.statusCode != 501) {
+                update(response.result.scene);
+                setLastAgentTarget(response.result.action);
+                updateScene[response.result.action].isAgentTarget = true;
+                setResItemNum(resItemNum - 1);
+            }
+            setExpired(response.statusCode === 501);
+            status = (response.statusCode === 501);
+        });
+
+        let updateQueue = queue.slice();
+        updateQueue.push(symbol);
+
+        setQueue(updateQueue);
+
+        setAnimating(true);
+        // console.timeEnd("process");
+        await waitTimeout(150);
+
+        const filterSame = updateQueue.filter((sb) => sb.icon === symbol.icon);
+
+        if (!status && filterSame.length === 3) {
+            updateQueue = updateQueue.filter((sb) => sb.icon !== symbol.icon);
+            for (const sb of filterSame) {
+                const find = updateScene.find((i) => i.id === sb.id);
+                if (find) {
+                    find.status = 2;
+                    if (soundRefMap.current) {
+                        soundRefMap.current[
+                            symbol.icon.tripleSound
+                        ].currentTime = 0;
+                        soundRefMap.current[symbol.icon.tripleSound].play();
+                    }
+                }
+            }
+        }
+
+        if (!status && updateQueue.length >= 7) {
+            setTipText('挑战失败');
+            setFinished(true);
+        }
+
+        if (!status && (!updateScene.find((s) => s.status === 0))) {
+            if (level === maxLevel) {
+                setTipText('全部关卡挑战成功');
+                setQueue([]);
+                setFinished(true);
+                setAnimating(false);
+                return;
+            }
+            setTipText('本关成功');
+            setLevel(level + 1);
+            setQueue([]);
+            restart(level + 1);
+        } else {
+            setQueue(updateQueue);
+        }
+
+        setAnimating(false);
+    };
+
+    return (
+        <>
+            <h2>DI-sheep: 深度强化学习 + 羊了个羊 </h2>
+            <h6>
+                <GithubIcon />
+            </h6>
+            <h3 className="flex-container flex-center">
+                主题:
+                <select
+                    onChange={(e) =>
+                        setCurTheme(themes[Number(e.target.value)])
+                    }
+                >
+                    {themes.map((t, idx) => (
+                        <option key={t.name} value={idx}>
+                            {t.name}
+                        </option>
+                    ))}
+                </select>
+            </h3>
+            <h3 className="flex-container flex-center">
+                关卡: {level}/{maxLevel}    用时: {(usedTime / 1000).toFixed(2)}秒    总牌数: {maxItemNum}
+                <br />
+                1-9关全部可消，第10关会额外设置2张单独的牌
+            </h3>
+
+            <div className="app">
+                <div className="scene-container">
+                    <div className="scene-inner">
+                        {scene.map((item, idx) => (
+                            <Symbol
+                                key={item.id}
+                                {...item}
+                                x={
+                                    item.status === 0
+                                        ? item.x
+                                        : item.status === 1
+                                        ? sortedQueue[item.id]
+                                        : -1000
+                                }
+                                y={item.status === 0 ? item.y : 895}
+                                onClick={() => clickSymbol(idx)}
+                            />
+                        ))}
+                    </div>
+                </div>
+            </div>
+            <div className="queue-container flex-container flex-center" />
+            <div className="flex-container flex-between">
+                <button className="flex-grow" onClick={() => restart(level)}>
+                    重开本关
+                </button>
+                <button className="flex-grow" onClick={levelUp}>
+                    下一关
+                </button>
+                <button className="flex-grow" onClick={() => clickSymbol(-1)}>
+                    AI 速通流
+                </button>
+                {/*<button onClick={test}>测试</button>*/}
+            </div>
+            <p style={{ textAlign: 'center' }}>
+                小提示：如果出现橙色背景块，这是 AI 提示的选择哦
+            </p>
+
+            {finished && (
+                <div className="modal">
+                    <h1>{tipText}</h1>
+                    <button onClick={() => restart(level)}> {expired ? '60s游戏过期，重来一关' : '再试试上一关'} </button>
+                </div>
+            )}
+
+            {/*bgm*/}
+            <button className="bgm-button" onClick={() => setBgmOn(!bgmOn)}>
+                {bgmOn ? '🔊' : '🔈'}
+                <audio ref={bgmRef} loop src="/song_of_kedaya.mp3" />
+            </button>
+
+            {/*音效*/}
+            {curTheme.sounds.map((sound) => (
+                <audio
+                    key={sound.name}
+                    ref={(ref) => {
+                        if (ref) soundRefMap.current[sound.name] = ref;
+                    }}
+                    src={sound.src}
+                />
+            ))}
+        </>
+    );
+};
+
+export default App;

DI-sheep/ui/src/GithubIcon.tsx

@@ -0,0 +1,28 @@
+import React, { FC } from 'react';
+
+export const GithubIcon: FC = () => {
+    return (
+        <a
+            href="https://github.com/orgs/opendilab/repositories"
+            target="_blank"
+            rel="noreferrer"
+        >
+            点个 Star 不迷路 @OpenDILab
+            <svg
+                height="24"
+                aria-hidden="true"
+                viewBox="0 0 16 16"
+                version="1.1"
+                width="24"
+                data-view-component="true"
+                className="octicon octicon-mark-github v-align-middle"
+                fill="currentColor"
+            >
+                <path
+                    fillRule="evenodd"
+                    d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"
+                />
+            </svg>
+        </a>
+    );
+};

DI-sheep/ui/src/index.css

@@ -0,0 +1,78 @@
+:root {
+    font-family: Inter, Avenir, Helvetica, Arial, sans-serif;
+    font-size: 16px;
+    line-height: 24px;
+    font-weight: 400;
+    color-scheme: light dark;
+    color: rgb(255 255 255 / 87%);
+    background-color: #242424;
+    font-synthesis: none;
+    text-rendering: optimizelegibility;
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+    text-size-adjust: 100%;
+}
+
+a {
+    font-weight: 500;
+    color: #646cff;
+    text-decoration: inherit;
+}
+
+a:hover {
+    color: #535bf2;
+}
+
+body {
+    margin: 0;
+    padding: 0 32px;
+}
+
+h1 {
+    font-size: 3.2em;
+    line-height: 1.1;
+}
+
+select {
+    border: 2px solid gray;
+    border-radius: 4px;
+    padding: 4px 8px;
+}
+
+button {
+    border-radius: 8px;
+    border: 1px solid transparent;
+    padding: 0.6em 1.2em;
+    font-size: 1em;
+    font-weight: 500;
+    font-family: inherit;
+    background-color: #1a1a1a;
+    cursor: pointer;
+    transition: border-color 0.25s;
+    word-break: keep-all;
+    outline: none;
+}
+
+button:hover {
+    border-color: #646cff;
+}
+
+button:focus,
+button:focus-visible {
+    outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+    :root {
+        color: #213547;
+        background-color: #fff;
+    }
+
+    a:hover {
+        color: #747bff;
+    }
+
+    button {
+        background-color: #f9f9f9;
+    }
+}

DI-sheep/ui/src/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+import './index.css';
+
+ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
+    <React.StrictMode>
+        <App />
+    </React.StrictMode>
+);

DI-sheep/ui/src/themes/default/index.ts

@@ -0,0 +1,40 @@
+import { Theme } from '../interface';
+
+const icons = <const>[
+    `🎨`,
+    `🌈`,
+    `⚙️`,
+    `💻`,
+    `📚`,
+    `🐯`,
+    `🐤`,
+    `🐼`,
+    `🐏`,
+    `🍀`,
+];
+
+export type DefaultSoundNames = 'button-click' | 'triple';
+
+import soundButtonClickUrl from './sounds/sound-button-click.mp3';
+import soundTripleUrl from './sounds/sound-triple.mp3';
+export const defaultSounds: Theme<DefaultSoundNames>['sounds'] = [
+    {
+        name: 'button-click',
+        src: soundButtonClickUrl,
+    },
+    {
+        name: 'triple',
+        src: soundTripleUrl,
+    },
+];
+
+export const defaultTheme: Theme<DefaultSoundNames> = {
+    name: '默认',
+    icons: icons.map((icon) => ({
+        name: icon,
+        content: icon,
+        clickSound: 'button-click',
+        tripleSound: 'triple',
+    })),
+    sounds: defaultSounds,
+};