{ "cells": [ { "cell_type": "markdown", "metadata": { "collapsed": true, "id": "brzvVeAsYiG2" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "metadata": { "id": "WMKrKfx8_3fc" }, "source": [ "# Instructor Grading and Assessment\n", "This notebook executes grading of student submissions of chats with ChatGPT, exported in JSON. Run each cell should be run in order, and follow the prompts displayed when appropriate." ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "696FqPrTYiG3", "outputId": "9679a415-8ab7-4c5f-e715-954d6801b6ec" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import ipywidgets as widgets\n", "from IPython.display import display, HTML, clear_output\n", "import io\n", "import zipfile\n", "import os\n", "import json\n", "import pandas as pd\n", "import glob\n", "from getpass import getpass" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "fTlnrMwmYiG4", "outputId": "e811e000-e9ec-43b6-d136-59d5134adeaf" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# \"global\" variables modified by mutability\n", "grade_settings = {'learning_objectives':None,\n", " 'json_file_path':None,\n", " 'json_files':None }" ] }, { "cell_type": "markdown", "metadata": { "id": "jb0jnIE14Vuh" }, "source": [ "The `InstructorGradingConfig` holds the contents of the instantiated object including making graindg settings, extracting files from a zip archive, loading JSON files into DataFrames, and displaying relevant information in the output widget." ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "mPLdaWiuYiG4", "outputId": "7a698bc1-7954-44ac-83c8-71d1dc410749" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "class InstructorGradingConfig:\n", " def __init__(self):\n", " # layouts to help with styling\n", " self.items_layout = widgets.Layout(width='auto')\n", "\n", " self.box_layout = widgets.Layout(display='flex',\n", " flex_flow='column',\n", " align_items='stretch',\n", " width='50%',\n", " border='solid 1px gray',\n", " padding='0px 30px 20px 30px')\n", "\n", " # Create all components\n", " self.ui_title = widgets.HTML(value=\"

Instructor Grading Configuration

\")\n", "\n", " self.run_button = widgets.Button(description='Submit', button_style='success', icon='check')\n", " self.status_output = widgets.Output()\n", " self.status_output.append_stdout('Waiting...')\n", "\n", " # Setup click behavior\n", " self.run_button.on_click(self._setup_environment)\n", "\n", " # Reset rest of state\n", " self.reset_state()\n", "\n", " def reset_state(self, close_all=False):\n", "\n", " if close_all:\n", " self.learning_objectives_text.close()\n", " self.file_upload.close()\n", " self.file_upload_box.close()\n", " #self.ui_container.close()\n", "\n", " self.learning_objectives_text = widgets.Textarea(value='', description='Learning Objectives',\n", " placeholder='Learning objectives: 1. Understand and implement classes in object-oriented programming',\n", " layout=self.items_layout,\n", " style={'description_width': 'initial'})\n", " self.file_upload = widgets.FileUpload(\n", " accept='.zip', # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'\n", " multiple=False # True to accept multiple files upload else False\n", " )\n", " self.file_upload_box = widgets.HBox([widgets.Label('Upload User Files:\\t'), self.file_upload])\n", "\n", "\n", " # Create a VBox container to arrange the widgets vertically\n", " self.ui_container = widgets.VBox([self.ui_title, self.learning_objectives_text,\n", " self.file_upload_box, self.run_button, self.status_output],\n", " layout=self.box_layout)\n", "\n", "\n", " def _setup_environment(self, btn):\n", " grade_settings['learning_objectives'] = self.learning_objectives_text.value\n", " grade_settings['json_file_path'] = self.file_upload.value\n", "\n", " if self.file_upload.value:\n", " try:\n", " input_file = list(self.file_upload.value.values())[0]\n", " extracted_zip_dir = list(grade_settings['json_file_path'].keys())[0][:-4]\n", " except:\n", " input_file = self.file_upload.value[0]\n", " extracted_zip_dir = self.file_upload.value[0]['name'][:-4]\n", "\n", " self.status_output.clear_output()\n", " self.status_output.append_stdout('Loading zip file...\\n')\n", "\n", " with zipfile.ZipFile(io.BytesIO(input_file['content']), \"r\") as z:\n", " z.extractall()\n", " extracted_files = z.namelist()\n", "\n", " self.status_output.append_stdout('Extracted files and directories: {0}\\n'.format(', '.join(extracted_files)))\n", "\n", " # load all json files\n", " grade_settings['json_files'] = glob.glob(''.join([extracted_zip_dir, '/**/*.json']), recursive=True)\n", "\n", " #status_output.clear_output()\n", " self.status_output.append_stdout('Loading successful!\\nLearning Objectives: {0}\\nExtracted JSON files: {1}'.format(grade_settings['learning_objectives'],\n", " ', '.join(grade_settings['json_files'])))\n", "\n", " else:\n", " self.status_output.clear_output()\n", " self.status_output.append_stdout('Please upload a zip file.')\n", "\n", " # Clear values so they're not saved\n", " self.learning_objectives_text.value = ''\n", " self.reset_state(close_all=True)\n", " self.run_ui_container()\n", "\n", " with self.status_output:\n", " print('Extracted files and directories: {0}\\n'.format(', '.join(extracted_files)))\n", " print('Loading successful!\\nLearning Objectives: {0}\\nExtracted JSON files: {1}'.format(grade_settings['learning_objectives'],\n", " ', '.join(grade_settings['json_files'])))\n", " print('Submitted and Reset all values.')\n", "\n", "\n", " def run_ui_container(self):\n", " display(self.ui_container, clear=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "4wCQ4Wk8YiG4", "outputId": "5c602e80-a210-4449-fd6c-eb8bf3213407" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "#This code helps in the case that we have problems with metadata being retained.\n", "#!jupyter nbconvert --ClearOutputPreprocessor.enabled=True --ClearMetadataPreprocessor.enabled=True --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"colab\" --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"kernelspec\" --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"language_info\" --to=notebook --output=instructor_inst_notebook.ipynb instructor_intr_notebook.ipynb" ] }, { "cell_type": "markdown", "metadata": { "id": "gj1K3MjHDlqb" }, "source": [ "# User Settings and Submission Upload\n", "The following two cells will ask you for your OpenAI API credentials and to upload the json file of the student submission." ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 519, "referenced_widgets": [ "a84d31fb8f4e4bafb74035158834b404", "b051a90758434644955747bc02d00bab", "252b8009f3734ed2908049ebb40c0247", "6622f76f91f44527a87a7575bbd388d2", "654ab6d155eb457ea5c719a9ac27ad5b", "86cb4f568f454ff8832face502fb0745", "e30fe87f01bc4580a61713b5b72439a2", "d16b25c7e9e948938c9303fbe8ae3dcc", "453b12da4b6540cd9e4e57f73a4d670c", "b74cf92175374028948d4cf529d4d1e6", "f7d75b0a32554a9589c513336fc30095", "7f7164e80a464ba9b99f96c10132db25", "49f80567705147f0b82d45b7f06dd1ba", "5a17f4509d194105b23dd616e45183d5", "81c4dda35a7d4e15821bb4bc0973354e", "df1c46361f714aceb9c046f98fede40c", "60b80d550efa403a825a3cb913c26f53", "d0bd0e3f12594ff1a51365b65a3fcc43", "dfa8d6c7d70b42468cbda035de89404c", "26d13984d45745858d3b890bc7f18a90", "53722998fbe64a7c94829b79e8cd69d6", "1b7ee0de15484cd5aecd6d8ca3b6ee9d", "dde20647d3594d31b66b19659f53a95e", "8610fffd2d2a4ec28f8c874c06073ce7", "54e3918921f44fb4a9020beab951fcdf", "1072a8a142f64dfd96ee528a2e9d1595", "67b4083cd4234f52bb7cca27ab9cddb3", "d0a1ebdf7fc0473f91c39b29ca580934", "abbecdc637694e7cb026e003244e7037", "7f814595d31e4b86992b5bd6bc85ced4", "76548751bb9c4bcb9d4f39788ea7d4af", "dbb88901f5084d49af208b91b52b6073" ] }, "id": "oQOeYl9OYiG5", "outputId": "bb5b7dc4-ea7b-41ea-e741-2fb2bf66cccc" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1b7ee0de15484cd5aecd6d8ca3b6ee9d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HTML(value='

Instructor Grading Configuration

'), Textarea(value='', description='Learni…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "InstructorGradingConfig().run_ui_container()" ] }, { "cell_type": "markdown", "metadata": { "id": "W9SqmkpeIgpk" }, "source": [ "You will need an OpenAI API key in order to access the chat functionality. In the following cell, you'll see a blank box pop up - copy your API key there and press enter." ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 32 }, "id": "MK8R5DmEYiG5", "outputId": "09e11ee6-5a9f-4b61-ff61-ddf82a68c498" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "··········\n" ] } ], "source": [ "# setup open AI api key\n", "openai_api_key = getpass()" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true, "id": "0bp158bj_0s6" }, "source": [ "# Execute Grading\n", "Run this cell set to have the generative AI assist you in grading." ] }, { "cell_type": "markdown", "metadata": { "id": "vyJuQ7RUR8tB" }, "source": [ "## Installation and Loading" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "tjKxWLA3YiG5", "outputId": "6dc85dff-4baa-44f0-edef-42925e6c271a" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%capture\n", "# install additional packages if needed\n", "! pip install -q langchain openai" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "S3oQiNm_YiG5", "outputId": "95e43744-fadc-45cc-bb02-05737d12fcb2" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# import necessary libraries here\n", "from langchain.llms import OpenAI\n", "from langchain.chat_models import ChatOpenAI\n", "from langchain.prompts import PromptTemplate\n", "from langchain.document_loaders import TextLoader\n", "from langchain.indexes import VectorstoreIndexCreator\n", "from langchain.text_splitter import CharacterTextSplitter\n", "from langchain.embeddings import OpenAIEmbeddings\n", "from langchain.schema import SystemMessage, HumanMessage, AIMessage\n", "import openai" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "uXfSTQPrYiG5", "outputId": "f85a6c4a-009f-4b30-f74a-04b6bfd85af6" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Helper because lines are printed too long; helps with wrapping visualization\n", "from IPython.display import HTML, display\n", "\n", "def set_css():\n", " display(HTML('''\n", " \n", " '''))\n", "get_ipython().events.register('pre_run_cell', set_css)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "sTQFW9TxYiG5", "outputId": "e291e167-e635-4006-965d-29fe1a0db10f" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Set pandas display options\n", "pd.set_option('display.max_columns', None)\n", "pd.set_option('display.max_colwidth', 0)" ] }, { "cell_type": "markdown", "metadata": { "id": "DOACT_LSSM58" }, "source": [ "Setting of API key in environment and other settings" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "OV05xRtDYiG5", "outputId": "0d6339d9-bc32-49e9-955f-99947b510456" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "#extract info from dictionary\n", "json_file_path = grade_settings['json_file_path']\n", "learning_objectives = grade_settings['learning_objectives']\n", "\n", "#set API key\n", "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n", "openai.api_key = openai_api_key" ] }, { "cell_type": "markdown", "metadata": { "id": "YreIs-I-tuxx" }, "source": [ "Initiate the OpenAI model using Langchain." ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "ZRn9wbJBYiG5", "outputId": "c09c7a7c-1ca0-4860-b39d-91778f183307" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "llm = ChatOpenAI(model='gpt-3.5-turbo-16k')\n", "messages = [\n", " SystemMessage(content=\"You are a helpful assistant.\"),\n", " HumanMessage(content=\"\")\n", "]" ] }, { "cell_type": "markdown", "metadata": { "id": "pIKYtr0UTJNc" }, "source": [ "## Functions to help with loading json" ] }, { "cell_type": "markdown", "metadata": { "id": "t7O3XPC29Osw" }, "source": [ "`file_upload_json_to_df` helps when you use the file uploader as the json is directly read in this case. `clean_keys` helps when there are errors on the keys when reading." ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "qGxPHexrYiG5", "outputId": "80657bb1-97e8-423a-a2ff-99afe8d22718" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Strip beginning and ending newlines\n", "def clean_keys(loaded_json):\n", " out_json = [{key.strip():value for key, value in json_dict.items()} for json_dict in loaded_json ]\n", " return out_json\n", "\n", "# Convert difficult datatypes to newlines\n", "def file_upload_json_to_df(upload_json):\n", "\n", " #get middle key of json to extract content\n", " fname = list(upload_json.keys())[0]\n", "\n", " #load the json; strict allows us to get around encoding issues\n", " loaded_json = json.loads(upload_json[fname]['content'], strict=False)\n", "\n", " #clean the keys if needed\n", " loaded_json = clean_keys(loaded_json)\n", "\n", " return pd.DataFrame(loaded_json)" ] }, { "cell_type": "markdown", "metadata": { "id": "N2yuYFQJYiG6" }, "source": [ "`create_user_dataframe` filters based on role to create a dataframe for only user responses" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 17 }, "id": "58hygjTXYiG6", "outputId": "8f3683fb-f3da-45f3-8338-772e7583d4cc" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def create_user_dataframe(df):\n", " df_user = df.query(\"`author` == 'user'\")\n", "\n", " return df_user" ] }, { "cell_type": "markdown", "metadata": { "id": "MOwaLI97Igpm" }, "source": [ "`load_json_as_df` helps when you use the file uploader as the json is directly read in this case. It accepts the path to the JSON to load the dataframe based on the json." ] }, { "cell_type": "code", "execution_count": 131, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "w0xN9CJeYiG6", "outputId": "9422452f-9a97-4f22-9e49-ea0812c298fd" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def load_json_as_df(fpath):\n", " # check if file is .json\n", " if not fpath.endswith('.json'):\n", " return None\n", "\n", " keys = [\"timestamp\", \"author\", \"message\"]\n", "\n", " df_out = None\n", " out_error = None\n", "\n", " try:\n", " # Read JSON file\n", " with open(fpath, \"r\") as f:\n", " json_data = f.read()\n", "\n", " # Load JSON data\n", " data = json.loads(json_data, strict=False)\n", "\n", " # Quick check to see if we can fix common errors in json\n", " # 1. JSON responses wrapped in enclosing dictionary\n", " if isinstance(data, dict):\n", " if len(data.keys()) == 1:\n", " data = data[list(data.keys())[0]]\n", " else:\n", " data = [data] # convert to list otherwise\n", "\n", " # We only operate on lists of dictionaries\n", " if isinstance(data, list):\n", " data = clean_keys(data) # clean keys to make sure there are no unnecessary newlines\n", "\n", " if all(all(k in d for k in keys) for d in data):\n", " # Filter only the student messages based on the \"author\" key\n", " data = [d for d in data if d[\"author\"].lower() == \"user\"]\n", "\n", " df_out = pd.json_normalize(data)\n", " if len(df_out) <= 1:\n", " out_error = [fpath, \"Warning: JSON keys correct, but something wrong with the overall structure of the JSON when converting to the dataframe. The dataframe only has one row. Skipping.\"]\n", " df_out = None\n", " else:\n", " out_error = [fpath, \"Error: JSON Keys are incorrect. Found keys: \" + str(list(data[0].keys()))]\n", " else:\n", " out_error = [fpath, \"Error: Something is wrong with the structure of the JSON.\"]\n", "\n", " except Exception as e:\n", " print(f\"Error processing file {fpath}: {str(e)}\")\n", " out_error = [fpath, \"Fatal System Error: \" + str(e)]\n", "\n", " if df_out is not None:\n", " df_out['filename'] = fpath\n", "\n", " return df_out, out_error" ] }, { "cell_type": "markdown", "metadata": { "id": "N2yuYFQJYiG6" }, "source": [ "`create_user_dataframe` filters based on role to create a dataframe for only user responses" ] }, { "cell_type": "code", "execution_count": 132, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "58hygjTXYiG6", "outputId": "44b588d6-6b6c-4c9e-b944-62ac29117344" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def create_user_dataframe(df):\n", " df_user = df.query(\"`author` == 'user'\")\n", "\n", " return df_user" ] }, { "cell_type": "markdown", "metadata": { "id": "KA5moX-1Igpn" }, "source": [ "The `process_file` and `process_files` functions provide the implementation of prompt templates for instructor grading. It uses the input components to assemble a prompt and then sends this prompt to the llm for evaluation alongside the read dataframes." ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "nFz3UVL3YiG6", "outputId": "01029e5f-b313-4941-d572-6dca5903d4ac" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def process_file(df, desc, instr, print_results):\n", " messages_as_string = '\\n'.join(df['message'].astype(str))\n", " context = messages_as_string\n", "\n", " # Assemble prompt\n", " prompt = desc if desc is not None else \"\"\n", " prompt = (prompt + instr + \"\\n\") if instr is not None else prompt\n", " prompt = prompt + \"Here is the chat log: \\n\\n\" + context + \"\\n\"\n", "\n", " # Get results and optionally print\n", " messages[1] = HumanMessage(content=prompt)\n", " result = llm(messages)\n", "\n", " # Check if 'filename' exists in df\n", " if 'filename' in df:\n", " if print_results:\n", " print(f\"\\n\\nResult for file {df['filename'][0]}: \\n{result.content}\")\n", " else:\n", " if print_results:\n", " print(f\"\\n\\nResult for file: Unknown Filename \\n{result.content}\")\n", "\n", " return result\n", "\n", "def process_files(json_dfs, output_desc=None, grad_instructions=None, use_defaults = False, print_results=True):\n", " if use_defaults:\n", " output_desc = (\"Given the following chat log, create a table with the question number, the question content, answer, \"\n", " \"whether or not the student answered correctly on the first try, and the number of attempts it took to get the right answer. \")\n", " grad_instructions = (\"Then, calculate the quiz grade from the total number of assessment questions. \"\n", " \"Importantly, a point should only be granted if an answer was correct on the very first attempt. \"\n", " \"If an answer was not correct on the first attempt, even if it was correct in subsequent attempts, no point should be awarded for that question. \")\n", "\n", " results = [process_file(df, output_desc, grad_instructions, print_results) for df in json_dfs]\n", "\n", " return results" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 17 }, "id": "EhryP8utrR9D", "outputId": "51f4a60d-d6b7-4885-85c4-410c741ed651" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def output_log_file(df_list, results_list, log_file='evaluation_log.txt'):\n", " \"\"\"\n", " Create a single log file containing evaluation results for all students.\n", "\n", " Parameters:\n", " df_list (list of pandas.DataFrame): List of DataFrames.\n", " results_list (list of ai_model_response): List of evaluation results.\n", " log_file (str): File name where the evaluation log will be saved. Default is 'evaluation_log.txt'.\n", "\n", " Returns:\n", " None\n", " \"\"\"\n", " with open(log_file, 'w') as log:\n", " for df, result in zip(df_list, results_list):\n", " log.write(f\"File: {df['filename'][0]}\\n\")\n", " log.write(result.content)\n", " log.write(\"\\n\\n\")" ] }, { "cell_type": "markdown", "metadata": { "id": "lXQ45cJ1AztR" }, "source": [ "`pretty_print` makes dataframes look better when printed by substituting non-HTML with HTML for rendering." ] }, { "cell_type": "code", "execution_count": 134, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "0te_RLOOYiG6", "outputId": "1dc53c98-5f68-4902-b377-7bba451395f0" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def pretty_print(df):\n", " return display( HTML( df.to_html().replace(\"\\\\n\",\"
\") ) )" ] }, { "cell_type": "markdown", "metadata": { "id": "I3rKk7lJYiG6" }, "source": [ "`save_as_csv` saves the dataframe as a CSV" ] }, { "cell_type": "code", "execution_count": 135, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "DnrH2ldeYiG6", "outputId": "f1f6153b-49db-4145-c188-373685ffdcf4" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def save_as_csv(df, file_name):\n", " df.to_csv(file_name, index=False)" ] }, { "cell_type": "code", "execution_count": 136, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "Vgo_y8R8bzTE", "outputId": "f5effec2-8620-4c1d-be15-672b8cb3de21" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def show_json_loading_errors(err_list):\n", " if err_list:\n", " print(\"The following files have the following errors upon loading and will NOT be processed:\", '\\n'.join(err_list))\n", " else:\n", " print(\"No errors found in uploaded zip JSON files.\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "85h5oTysJkHs" }, "source": [ "## Final data preparation steps" ] }, { "cell_type": "code", "execution_count": 137, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "Upah5_ygZRZx", "outputId": "639a0d70-6a93-4462-f65a-2e24f30643c0" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "#additional processing setup\n", "json_files = grade_settings['json_files']\n", "load_responses = [load_json_as_df(jf) for jf in json_files]\n", "\n", "#unzip to two separate lists\n", "all_json_dfs, errors_list = zip(*load_responses)\n", "\n", "# Remove failed JSONs\n", "all_json_dfs = [df for df in all_json_dfs if df is not None]\n", "\n", "# Update errors list to be individual strings\n", "errors_list = [' '.join(err) for err in errors_list if err is not None]" ] }, { "cell_type": "markdown", "metadata": { "id": "P_H4uIfmAsr0" }, "source": [ "# AI-Assisted Evaluation\n", "Introduction and Instructions\n", "--------------------------------------------------\n", "The following example illustrates how you can specify important components of the prompts for sending to the llm. The `process_files` function will iterate over all of the submissions in your zip file, create dataframes of results (via instruction by setting `output_setup`), and also perform evaluation based on your instructions (via instruction by setting `grading_instructions`).\n", "\n", "Example functionality is demonstrated below." ] }, { "cell_type": "code", "execution_count": 138, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 35 }, "id": "9zIPjG5lco3Z", "outputId": "cc9531c0-3939-4c9f-dc6c-9d2cfa0ea7d2" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "No errors found in uploaded zip JSON files.\n" ] } ], "source": [ "# Print list of files with the incorrect format\n", "show_json_loading_errors(errors_list)" ] }, { "cell_type": "code", "execution_count": 139, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "id": "utPzYUoKYiG9", "outputId": "eb3e1769-eb7a-4c93-96bc-6c998df55ef1" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "Result for file instructorTest/spencer-smith_jesse.json: \n", "Summary and feedback for student responses:\n", "\n", "Student 1:\n", "The student provided an excellent response to Question 1. They accurately explained the purpose of capitalizing expenses when incorporating them into the estimate of corporate earnings. They highlighted the importance of accurately reflecting the timing of costs and their related benefits, and how capitalizing expenses can impact a company's financial statements. The student also mentioned the matching principle of accounting and its role in ensuring the comparability and fairness of financial statements. Overall, the response is comprehensive and well-written. Well done!\n", "\n", "Student 2:\n", "The student gave a great response to Question 2. They correctly stated that expenses should be capitalized when they provide value beyond the current accounting period. The student also provided examples of capital expenses, such as the purchase price of a delivery truck or the cost of a building renovation. These examples demonstrate a clear understanding of the topic. The response is well-explained and shows a good grasp of the concept. Great job!\n", "\n", "Numeric summary:\n", "Both students provided correct answers to their respective questions, earning them a point each. Therefore, the numeric summary is as follows:\n", "Student 1: 1 point\n", "Student 2: 1 point\n", "\n", "\n", "Result for file instructorTest/bell_charreau.json: \n", "Summary:\n", "- The first student's response inaccurately states that capitalizing expenses is done to make the money 'look good' on the earnings report. The assistant provides a detailed explanation of the correct purpose of capitalizing expenses.\n", "- The second student's response partially identifies that capitalized expenses provide benefits for a longer period, but the assistant provides a more comprehensive explanation of what types of expenses should be capitalized and why they are treated differently from regular expenses.\n", "\n", "Feedback for the first student:\n", "The student accurately identified the purpose of capitalizing expenses, but their explanation was not entirely correct. They incorrectly stated that it is done to make the money 'look good' on the earnings report. The assistant provided a clear and detailed explanation of the correct purpose of capitalizing expenses and how it aligns with accounting principles.\n", "\n", "Feedback for the second student:\n", "The student partially identified the types of expenses that should be capitalized and why they are treated differently from regular expenses. However, their explanation was not comprehensive. The assistant provided a more detailed explanation of what types of expenses should be capitalized and why, as well as the concept of depreciation or amortization for spreading out the costs over time.\n", "\n", "Numeric Summary:\n", "The first student's response was partially correct, so they receive 0.5 points.\n", "The second student's response was also partially correct, so they receive 0.5 points.\n", "The total point count is 1.\n" ] } ], "source": [ "# Example\n", "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n", " \", including what the student did well, and what was done poorly. \"\n", " \"Additionally, please filter feedback alphabetically by the name of the student from the filename.\")\n", "grading_instructions = (\"Then, calculate a numeric summary, summing up the point totals, \"\n", " \"in which a point is awarded for answering correctly. \")\n", "\n", "# Assuming `file_paths` is a list of file paths.\n", "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n", "\n", "output_log_file(all_json_dfs, processed_submissions)" ] }, { "cell_type": "markdown", "metadata": { "id": "Pc1myGweIgpo" }, "source": [ "## Instructor-Specified Evaluation\n", "Now, you can use the following code to create your settings. Change `output_setup` and `grading_instructions` as desired, making sure to keep the syntax (beginning and ending parentheses,and quotes at the beginning and end of each line) correct. `output_setup` has been copied from the previous cell, but you should fill in `grading_instructions`.\n", "\n", "### File Processing Options\n", "The `process_files` function has a number of settings.\n", "* The first setting must always be `all_json_dfs`, which contains the tabular representation of the json output.\n", "* The other settings should be set by name, and are:\n", " * **`output_desc`**: Shown as `output_setup` here, this contains the isntructions about how you want to the tabular representation to be set up. Note that you can also leave this off of the function list (just erase it and the following comma).\n", " * **`grad_instructions`**: Shown as `grading_instructions` here, use this variable to set grading instructions. Note that you can also leave this off of the function list (erase it and the following comma)\n", " * **`use_defaults`**: Some default grading and instruction prompts have already been created. If you set `use_defaults=TRUE`, both the grading instructions and the output table description will use the default prompts provided by the program, regardless of whether you have set values for `output_desc` or `grad_instructions`.\n", " * **`print_results`**: By default, the results will be printed for all students. However, if you don't want to see this output, you can set `print_results=False`.\n", "\n", "Again, make sure to observe the syntax. The defaults used in the program are shown in the above example." ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "GiebKVlbYiG9", "outputId": "d4769d23-393c-4986-9418-5cef6944e6ab" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n", " \", including what the student did well, and what was done poorly. \")\n", "\n", "# add your own grading instructions\n", "grading_instructions = (\"INSERT ANY CUSTOM GRADING INSTRUCTIONS HERE\")\n", "\n", "# Assuming `file_paths` is a list of file paths.\n", "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n", "\n", "output_log_file(all_json_dfs, processed_submissions)" ] }, { "cell_type": "markdown", "metadata": { "id": "snLA6OZ83CrS" }, "source": [ "## Grading based on Blooms Taxonomy\n", "Another mechanism of evaluation is through Bloom's Taxonomy, where student responses will be evaluated based on where they fall on Bloom's Taxonomy. The higher the score with Bloom's Taxonomy, the more depth is illustrated by the question." ] }, { "cell_type": "code", "execution_count": 140, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 625 }, "id": "HEPXCJdrYiG-", "outputId": "add813e2-6b7c-4772-dac5-5f756e893b8f" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "Result for file 0 instructorTest/bell_charreau.json \n", "0 instructorTest/spencer-smith_jesse.json\n", "Name: filename, dtype: object: \n", "Student 1:\n", "Summary: The student incorrectly states that capitalizing expenses is done to make the money look good on the earnings report.\n", "Feedback: The student's response is not accurate. They misunderstood the purpose of capitalizing expenses. The main purpose is to spread the cost of certain long-term assets over their useful life, not to make the money 'look good' on the earnings report. \n", "Overall Level of Engagement and Knowledge: 1 (Remember)\n", "\n", "Student 2:\n", "Summary: The student partially understands the purpose of capitalizing expenses, but their answer could be more comprehensive.\n", "Feedback: The student correctly notes that capitalized expenses provide benefits for a longer period and are different from regular expenses. However, their answer could be more comprehensive and provide a more thorough explanation of why certain expenses are capitalized and how they are treated differently.\n", "Overall Level of Engagement and Knowledge: 3 (Apply)\n", "\n", "Student 3:\n", "Summary: The student provides a comprehensive and accurate explanation of the purpose of capitalizing expenses.\n", "Feedback: The student's response is excellent. They provide a comprehensive explanation of why capitalizing expenses is crucial for incorporating costs into the estimate of corporate earnings. They correctly highlight the timing of costs and their related benefits, the impact on financial statements, and the alignment with the matching principle of accounting.\n", "Overall Level of Engagement and Knowledge: 6 (Create)\n", "\n", "Student 4:\n", "Summary: The student accurately states that expenses should be capitalized when they provide value beyond the current accounting period and provides relevant examples.\n", "Feedback: The student's response is great. They correctly identify the types of expenses that should be capitalized and provide relevant examples. They demonstrate a clear understanding of the concept and provide a well-explained answer.\n", "Overall Level of Engagement and Knowledge: 5 (Evaluate)\n" ] } ], "source": [ "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n", " \", including what the student did well, and what was done poorly. \")\n", "grading_instructions = \"\"\"\\nEvaluate the each student's overall level or engagement and knowledge, based on bloom's taxonomy using their responses.\n", "Bloom's taxonomy is rated on a 1-6 point system, with 1 being remember (recall facts and basic concepts), 2 being understand (explain ideas or concepts),\n", "3 being apply (use information in new situations), 4 being analyze (draw connections among ideas), 5 being evaluate (justify a stand or decision),\n", "and 6 being create (produce new or original work). Assign the interaction a score from 1-6, where 1 = remember, 2 = understand, 3 = apply, 4 = analyze,\n", "5 = evaluate, and 6 = create.\"\"\"\n", "\n", "# Assuming `file_paths` is a list of file paths.\n", "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n", "\n", "output_log_file(all_json_dfs, processed_submissions)" ] }, { "cell_type": "markdown", "metadata": { "id": "FI5-vnUvXM03" }, "source": [ "# Returning Results\n" ] }, { "cell_type": "markdown", "metadata": { "id": "LgoGt82CYiG-" }, "source": [ "**Extract Student Responses ONLY from CHAT JSON**\n", "\n", "Below are relevant user components of dataframes, including the conversion from the original json, the interaction labeled dataframe, and the output dataframe. Check to make sure they make sense." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "HVq9i_mXYiG-", "outputId": "fb5251e9-a327-4dfa-e294-03f5c58f6d35" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def write_responses_to_csv(json_dfs):\n", " # Concatenate all dataframes in json_dfs into one large dataframe\n", " df = pd.concat(json_dfs)\n", "\n", " # Write the dataframe to a CSV\n", " df.to_csv('all_student_responses.csv', index=False)\n", "\n", "write_responses_to_csv(all_json_dfs)" ] }, { "cell_type": "markdown", "metadata": { "id": "1WIGxKmDYiG-" }, "source": [ "**Saving/Downloading AI-Assisted Student Evaluation from Chat JSON**\n", "\n", "Execute the following cell to have all of your students' data returned in a single CSV file." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 16 }, "id": "QnWNEeqjYiG-", "outputId": "e98c0c39-8449-45d4-f93a-d9394f6781bf" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Start with an empty dataframe\n", "all_results_df = pd.DataFrame()\n", "\n", "for result in processed_submissions:\n", "\n", " # Append the data from the current file to the master dataframe\n", " all_results_df = pd.concat([all_results_df, df])\n", "\n", "# Now all_results_df contains data from all the files\n", "\n", "# Write all results to a single CSV\n", "all_results_df.to_csv('all_results.csv', index=False)" ] } ], "metadata": { "colab": { "include_colab_link": true, "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.10.6" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "1072a8a142f64dfd96ee528a2e9d1595": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "LabelModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "LabelModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "LabelView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_abbecdc637694e7cb026e003244e7037", "placeholder": "​", "style": "IPY_MODEL_7f814595d31e4b86992b5bd6bc85ced4", "value": "Upload User Files:\t" } }, "1b7ee0de15484cd5aecd6d8ca3b6ee9d": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "VBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "VBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "VBoxView", "box_style": "", "children": [ "IPY_MODEL_b051a90758434644955747bc02d00bab", "IPY_MODEL_dde20647d3594d31b66b19659f53a95e", "IPY_MODEL_8610fffd2d2a4ec28f8c874c06073ce7", "IPY_MODEL_654ab6d155eb457ea5c719a9ac27ad5b", "IPY_MODEL_86cb4f568f454ff8832face502fb0745" ], "layout": "IPY_MODEL_e30fe87f01bc4580a61713b5b72439a2" } }, "252b8009f3734ed2908049ebb40c0247": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "TextareaModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "TextareaModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "TextareaView", "continuous_update": true, "description": "Learning Objectives", "description_tooltip": null, "disabled": false, "layout": "IPY_MODEL_b74cf92175374028948d4cf529d4d1e6", "placeholder": "Learning objectives: 1. Understand and implement classes in object-oriented programming", "rows": null, "style": "IPY_MODEL_f7d75b0a32554a9589c513336fc30095", "value": "" } }, "26d13984d45745858d3b890bc7f18a90": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ButtonStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "button_color": null, "font_weight": "" } }, "453b12da4b6540cd9e4e57f73a4d670c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "49f80567705147f0b82d45b7f06dd1ba": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FileUploadModel", "state": { "_counter": 1, "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FileUploadModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "FileUploadView", "accept": ".zip", "button_style": "", "data": [ null ], "description": "Upload", "description_tooltip": null, "disabled": false, "error": "", "icon": "upload", "layout": "IPY_MODEL_dfa8d6c7d70b42468cbda035de89404c", "metadata": [ { "lastModified": 1689919477171, "name": "instructorTest.zip", "size": 4958, "type": "application/zip" } ], "multiple": false, "style": "IPY_MODEL_26d13984d45745858d3b890bc7f18a90" } }, "53722998fbe64a7c94829b79e8cd69d6": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "54e3918921f44fb4a9020beab951fcdf": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "initial" } }, "5a17f4509d194105b23dd616e45183d5": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "60b80d550efa403a825a3cb913c26f53": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "654ab6d155eb457ea5c719a9ac27ad5b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ButtonModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ButtonView", "button_style": "success", "description": "Submit", "disabled": false, "icon": "check", "layout": "IPY_MODEL_81c4dda35a7d4e15821bb4bc0973354e", "style": "IPY_MODEL_df1c46361f714aceb9c046f98fede40c", "tooltip": "" } }, "6622f76f91f44527a87a7575bbd388d2": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_7f7164e80a464ba9b99f96c10132db25", "IPY_MODEL_49f80567705147f0b82d45b7f06dd1ba" ], "layout": "IPY_MODEL_5a17f4509d194105b23dd616e45183d5" } }, "67b4083cd4234f52bb7cca27ab9cddb3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FileUploadModel", "state": { "_counter": 0, "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FileUploadModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "FileUploadView", "accept": ".zip", "button_style": "", "data": [], "description": "Upload", "description_tooltip": null, "disabled": false, "error": "", "icon": "upload", "layout": "IPY_MODEL_76548751bb9c4bcb9d4f39788ea7d4af", "metadata": [], "multiple": false, "style": "IPY_MODEL_dbb88901f5084d49af208b91b52b6073" } }, "76548751bb9c4bcb9d4f39788ea7d4af": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7f7164e80a464ba9b99f96c10132db25": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "LabelModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "LabelModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "LabelView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_60b80d550efa403a825a3cb913c26f53", "placeholder": "​", "style": "IPY_MODEL_d0bd0e3f12594ff1a51365b65a3fcc43", "value": "Upload User Files:\t" } }, "7f814595d31e4b86992b5bd6bc85ced4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "81c4dda35a7d4e15821bb4bc0973354e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8610fffd2d2a4ec28f8c874c06073ce7": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_1072a8a142f64dfd96ee528a2e9d1595", "IPY_MODEL_67b4083cd4234f52bb7cca27ab9cddb3" ], "layout": "IPY_MODEL_d0a1ebdf7fc0473f91c39b29ca580934" } }, "86cb4f568f454ff8832face502fb0745": { "model_module": "@jupyter-widgets/output", "model_module_version": "1.0.0", "model_name": "OutputModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/output", "_model_module_version": "1.0.0", "_model_name": "OutputModel", "_view_count": null, "_view_module": "@jupyter-widgets/output", "_view_module_version": "1.0.0", "_view_name": "OutputView", "layout": "IPY_MODEL_53722998fbe64a7c94829b79e8cd69d6", "msg_id": "", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Extracted files and directories: instructorTest/, __MACOSX/._instructorTest, instructorTest/bell_charreau.json, __MACOSX/instructorTest/._bell_charreau.json, instructorTest/spencer-smith_jesse.json, __MACOSX/instructorTest/._spencer-smith_jesse.json\n", "\n", "Loading successful!\n", "Learning Objectives: \n", "Extracted JSON files: instructorTest/spencer-smith_jesse.json, instructorTest/bell_charreau.json\n", "Submitted and Reset all values.\n" ] } ] } }, "a84d31fb8f4e4bafb74035158834b404": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "VBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "VBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "VBoxView", "box_style": "", "children": [ "IPY_MODEL_b051a90758434644955747bc02d00bab", "IPY_MODEL_252b8009f3734ed2908049ebb40c0247", "IPY_MODEL_6622f76f91f44527a87a7575bbd388d2", "IPY_MODEL_654ab6d155eb457ea5c719a9ac27ad5b", "IPY_MODEL_86cb4f568f454ff8832face502fb0745" ], "layout": "IPY_MODEL_e30fe87f01bc4580a61713b5b72439a2" } }, "abbecdc637694e7cb026e003244e7037": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b051a90758434644955747bc02d00bab": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d16b25c7e9e948938c9303fbe8ae3dcc", "placeholder": "​", "style": "IPY_MODEL_453b12da4b6540cd9e4e57f73a4d670c", "value": "

Instructor Grading Configuration

" } }, "b74cf92175374028948d4cf529d4d1e6": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": "auto" } }, "d0a1ebdf7fc0473f91c39b29ca580934": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d0bd0e3f12594ff1a51365b65a3fcc43": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d16b25c7e9e948938c9303fbe8ae3dcc": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dbb88901f5084d49af208b91b52b6073": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ButtonStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "button_color": null, "font_weight": "" } }, "dde20647d3594d31b66b19659f53a95e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "TextareaModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "TextareaModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "TextareaView", "continuous_update": true, "description": "Learning Objectives", "description_tooltip": null, "disabled": false, "layout": "IPY_MODEL_b74cf92175374028948d4cf529d4d1e6", "placeholder": "Learning objectives: 1. Understand and implement classes in object-oriented programming", "rows": null, "style": "IPY_MODEL_54e3918921f44fb4a9020beab951fcdf", "value": "" } }, "df1c46361f714aceb9c046f98fede40c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ButtonStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "button_color": null, "font_weight": "" } }, "dfa8d6c7d70b42468cbda035de89404c": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e30fe87f01bc4580a61713b5b72439a2": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": "stretch", "align_self": null, "border": "solid 1px gray", "bottom": null, "display": "flex", "flex": null, "flex_flow": "column", "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": "0px 30px 20px 30px", "right": null, "top": null, "visibility": null, "width": "50%" } }, "f7d75b0a32554a9589c513336fc30095": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "initial" } } } } }, "nbformat": 4, "nbformat_minor": 0 }