notebooks/analyze_survey_responses.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Survey Response Analysis\n",
    "---\n",
    "Author: Peter Zhang\n",
    "\n",
    "This notebook performs the core survey analysis for the results in the paper."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import packages, read cleaned data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import datetime as dt\n",
    "from itertools import product\n",
    "from os.path import join\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mtick\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import ttest_ind, pearsonr, brunnermunzel\n",
    "import seaborn as sns\n",
    "from sklearn.calibration import calibration_curve\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from statsmodels.iolib.summary2 import summary_col\n",
    "from statsmodels.regression.linear_model import OLS\n",
    "from statsmodels.stats.multitest import fdrcorrection\n",
    "from tqdm import tqdm\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "import config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.dpi'] = 300\n",
    "plt.rcParams['savefig.dpi'] = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = \"../data/cleaned_responses\"\n",
    "participant_df = pd.read_csv(join(data_folder, \"responses_0425_responses.csv\"))\n",
    "question_df = pd.read_csv(join(data_folder, \"responses_0425_features.csv\"))\n",
    "\n",
    "# printout info\n",
    "question_num = len(question_df)\n",
    "print(f\"The dataset includes {question_num} observations.\")\n",
    "participant_num = len(participant_df)\n",
    "print(f\"There were {participant_num} unique participants.\")\n",
    "unique_questions_num = len(question_df['question'].unique())\n",
    "print(f\"Participants were shown {unique_questions_num} unique questions.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA\n",
    "\n",
    "From the section `Appendix B.1`."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary Statistics"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Counts of participants in each condition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(question_df.groupby(['advisor', 'give_justification']).agg({'participant_id':'nunique'}))\n",
    "print(f\"Participants answered an average of {question_num/participant_num:.2f} questions.\")\n",
    "print(f\"{sum(question_df.groupby('participant_id').agg({'question_num':'nunique'})['question_num'] == 40)} participants answered all 40 questions.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates tables of summary statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summary stat columns\n",
    "# woa, init advice conf, advice confidence, advisor, give justification, advice correct\n",
    "column_groups = {\n",
    "    \"advice\":[\"weight_on_advice\", \"init_advice_confidence\", \"advice_confidence\", \"advice_is_correct\"],\n",
    "    \"familiarity\":[\"net_familiarity\", \"uncomfortable\", \"neutral\", \"comfortable\"],\n",
    "    \"usage\":[\"usage_level\", \"heard_of\", \"used\", \"used_in_class\", \"answered_mc\"],\n",
    "    \"experience_time\":[\"question_num\", \"correct_advice_count\", \"incorrect_advice_count\",\"init_time\", \"adjusted_time\"]\n",
    "}\n",
    "\n",
    "for type, columns in column_groups.items():\n",
    "    summary_df = question_df[columns].copy()\n",
    "\n",
    "    # round to 2 decimals\n",
    "    summary_df.describe().round(3).to_latex(f\"../tables/summary_stats_{type}.tex\")\n",
    "    print(summary_df.describe().round(3).to_string())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For `Appendix A.1` on question distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_df[\"participant+question_id\"] = question_df[\"participant_id\"].astype(str) + question_df[\"question_id\"].astype(str)\n",
    "plt.hist(list(question_df['question_id'].value_counts()), bins=max(question_df['question_id'].value_counts() - 1))\n",
    "plt.xlabel(\"Number of Apperances\")\n",
    "plt.ylabel(\"Number of Questions\")\n",
    "plt.title(\"Distribution of Questions\")\n",
    "plt.savefig('../figures/question_distribution.png')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot distribution of weight on advice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Average WoA was {question_df['weight_on_advice'].mean():.3f} with a standard deviation of {question_df['weight_on_advice'].std():.3f}\")\n",
    "print(f\"Proportion of WoA set to 0: {(question_df['weight_on_advice'] == 0).mean():.2%}\")\n",
    "question_df['weight_on_advice'].plot.hist(bins=20)\n",
    "plt.title(\"Distribution of Weight on Advice\")\n",
    "plt.xlabel(\"Weight on Advice\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.savefig(\"../figures/weight_on_advice_dist.png\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question Count"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot dropoff in questions answered after 20."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Participants answered an average of {question_df.groupby('participant_id')['question_num'].count().mean():.1f} questions.\")\n",
    "print(f\"Number of participant who answered 25 questions or less: {len(question_df.groupby('participant_id').filter(lambda x: len(x) <= 25)['participant_id'].unique())}\")\n",
    "print(f\"Number of participant who answered more than 25 questions: {len(question_df.groupby('participant_id').filter(lambda x: len(x) > 25)['participant_id'].unique())}\")\n",
    "sns.histplot(data=question_df, x=\"question_num\", bins=np.arange(1, 42))\n",
    "plt.xlabel(\"Question Number\")\n",
    "plt.title(\"Questions Answered\")\n",
    "plt.savefig('../figures/questions_answered.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot average weight on advice through the experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.pointplot(data=question_df, x=\"question_group\", y=\"weight_on_advice\", errorbar=('ci',95))\n",
    "plt.title('Weight on Advice Through the Experiment')\n",
    "plt.xlabel('Question Number')\n",
    "g.set_xticklabels(labels=[f'{i}-{i+4}' for i in range(1, 41, 5)])\n",
    "plt.ylabel('Weight on Advice\\n(95% CI)')\n",
    "plt.savefig('../figures/over_questions.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Analysis of variance for question group."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = smf.ols('weight_on_advice ~ C(question_group)', data=question_df).fit()\n",
    "anova_table = sm.stats.anova_lm(model, typ=2)\n",
    "anova_table"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Participant majors."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Familiarity\n",
    "\n",
    "From the section `Appendix B.2`. First calculate overall accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pick_answer(row: pd.Series, type=str) -> str:\n",
    "    \"\"\"Pick answer based on confidence and randomly tiebreak.\"\"\"\n",
    "    assert type in ('init', 'adjusted', 'simulated'), 'type must be init or adjusted'\n",
    "\n",
    "    random.seed(0)\n",
    "    choices = [row[f'{type}_choice{letter}'] for letter in ['A', 'B', 'C', 'D']]\n",
    "    max_conf = max(choices)\n",
    "    choices = [letter for letter, conf in zip(['A', 'B', 'C', 'D'], choices) if conf == max_conf]\n",
    "    return random.choice(choices)\n",
    "\n",
    "question_df['init_answer'] = question_df.apply(pick_answer, axis=1, type='init')\n",
    "question_df['init_answer_correct'] = question_df['init_answer'] == question_df['correct_answer']\n",
    "question_df['adjusted_answer'] = question_df.apply(pick_answer, axis=1, type='adjusted')\n",
    "question_df['adjusted_answer_correct'] = question_df['adjusted_answer'] == question_df['correct_answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ANSWER_COLS = ['init_answer_correct', 'adjusted_answer_correct', 'advice_is_correct']\n",
    "COMFORT_LEVELS = [\"Combined\", 'Uncomfortable', 'Neutral', 'Comfortable']\n",
    "def eval_accuracy(df, comfort_level, answer_col):\n",
    "    \"\"\"Evaluates the mean and standard error of accuracy for a given comfort level.\"\"\"\n",
    "    assert answer_col in ANSWER_COLS\n",
    "    assert comfort_level in COMFORT_LEVELS\n",
    "\n",
    "    if comfort_level != \"Combined\":\n",
    "        df = df[df['topic_familiarity'] == comfort_level]\n",
    "    \n",
    "    accuracy = df[answer_col].mean()\n",
    "    se = df[answer_col].sem()\n",
    "\n",
    "    return accuracy, se"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate and plot\n",
    "accuracy_data = []\n",
    "for answer_col, comfort_level in product(ANSWER_COLS, COMFORT_LEVELS):\n",
    "    accuracy, se = eval_accuracy(question_df, comfort_level, answer_col)\n",
    "    accuracy_data.append({'answer_col': answer_col, 'comfort_level': comfort_level, 'accuracy': accuracy, 'se': se})\n",
    "accuracy_df = pd.DataFrame(accuracy_data, columns=['answer_col', 'comfort_level', 'accuracy', 'se'])\n",
    "\n",
    "# evaluate which accuracies are significantly different from combined\n",
    "sig_diff = []\n",
    "for answer_col in ANSWER_COLS:\n",
    "    combined_accuracy = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == 'Combined')]['accuracy'].values[0]\n",
    "    for comfort_level in COMFORT_LEVELS:\n",
    "        accuracy = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == comfort_level)]['accuracy'].values[0]\n",
    "        se = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == comfort_level)]['se'].values[0] * 1.96\n",
    "        if accuracy - se > combined_accuracy or accuracy + se < combined_accuracy:\n",
    "            sig_diff.append((answer_col, comfort_level))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display accuracy by topic familiarity level."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8, 6))\n",
    "sns.barplot(data=accuracy_df, x='comfort_level', y='accuracy', hue='answer_col', ax=ax)\n",
    "ax.set_ylim(0, 1)\n",
    "ax.set_ylabel('Accuracy\\n(95% CI, p<0.05)')\n",
    "ax.set_xlabel('Topic Familiarity')\n",
    "ax.set_title('Topic Familiarity Influences Accuracy')\n",
    "\n",
    "# plot error bars\n",
    "for i, answer_col in enumerate(ANSWER_COLS):\n",
    "    for j, comfort_level in enumerate(COMFORT_LEVELS):\n",
    "        accuracy = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == comfort_level)]['accuracy'].values[0]\n",
    "        se = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == comfort_level)]['se'].values[0] * 1.96\n",
    "        ax.errorbar(j + i * 0.26 - 0.26, accuracy, yerr=se, fmt='none', color='black', capsize=5)\n",
    "\n",
    "# draw horizontal lines for combined accuracy\n",
    "for i, answer_col in enumerate(ANSWER_COLS):\n",
    "    combined_accuracy = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == 'Combined')]['accuracy'].values[0]\n",
    "    color = {\n",
    "        'init_answer_correct': 'tab:blue',\n",
    "        'adjusted_answer_correct': 'tab:orange',\n",
    "        'advice_is_correct': 'tab:green'\n",
    "    }[answer_col]\n",
    "    ax.axhline(combined_accuracy, color=color, linestyle='--', alpha=0.5)\n",
    "\n",
    "# annotate accuracy at top of each bar\n",
    "for i, answer_col in enumerate(ANSWER_COLS):\n",
    "    for j, comfort_level in enumerate(COMFORT_LEVELS):\n",
    "        accuracy = accuracy_df[(accuracy_df['answer_col'] == answer_col) & (accuracy_df['comfort_level'] == comfort_level)]['accuracy'].values[0]\n",
    "\n",
    "        # bold if significantly different from combined\n",
    "        fontweight = 'bold' if (answer_col, comfort_level) in sig_diff else 'normal'\n",
    "        ax.annotate(f\"{accuracy * 100:.1f}\", xy=(j + i * 0.26 - 0.26, accuracy + 0.05), ha='center', fontsize=12, fontweight=fontweight)\n",
    "\n",
    "# rename legend\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "ax.legend(handles, ['Initial', 'Adjusted', 'Advice'], title='Answer Type', loc='upper left')\n",
    "\n",
    "ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))\n",
    "plt.tight_layout()\n",
    "plt.savefig('../figures/accuracy_by_answer_type_and_comfort_level.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comfort summary statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fam_topic_data = []\n",
    "for i, topic in enumerate(config.fam_topics):\n",
    "    for c in participant_df[f\"familiarity_{i+1}\"]:\n",
    "        fam_topic_data.append({\n",
    "            \"fam_topic\":topic,\n",
    "            \"net_familiarity\":config.comfort_map[c]\n",
    "        })\n",
    "fam_topic_df = pd.DataFrame(fam_topic_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_topics = fam_topic_df.groupby('fam_topic')['net_familiarity'].mean().sort_values().index"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot familiarity by topic area. Figure from `Appendix B.1`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_topics = fam_topic_df.groupby('fam_topic')['net_familiarity'].mean().sort_values().index\n",
    "sns.pointplot(x='net_familiarity', y='fam_topic', data=fam_topic_df, hue_order=COMFORT_LEVELS, palette=\"coolwarm\", order=sorted_topics, orient='h')\n",
    "# annotate net mean\n",
    "for i, label in enumerate(plt.gca().get_yticklabels()):\n",
    "    mean_net_comfort = fam_topic_df[fam_topic_df['fam_topic'] == label._text]['net_familiarity'].mean()\n",
    "    color = sns.color_palette(\"coolwarm\")[round(5*mean_net_comfort) + 2]\n",
    "    plt.gca().text(-.6, i-.13, f\"{mean_net_comfort:.3f}\", ha='center', color=color)\n",
    "plt.gca().axvline(0, color='grey', linestyle='--', linewidth=1)  # add thin zero line\n",
    "# pad top\n",
    "plt.gca().set_yticklabels(sorted_topics)\n",
    "\n",
    "plt.ylabel('Topic')\n",
    "plt.xlabel('Net Familiarity\\n(95% CI)')\n",
    "plt.title('Familiarity with Topic Areas')\n",
    "plt.tight_layout()\n",
    "plt.savefig('../figures/familiarity.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Usage"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot weight on advice by usage level for each displayed advisor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for advisor in (\"expert\", \"AI chatbot\"):\n",
    "    condition_name = \"AI Chatbot\" if advisor == 'AI chatbot' else \"Human Expert\"\n",
    "    # plot woa by usage level\n",
    "    advisor_df = question_df[question_df['advisor'] == advisor].copy()\n",
    "    plt.figure(figsize=(6, 6))\n",
    "    sns.pointplot(x='usage_level', y='weight_on_advice', data=advisor_df, errorbar=('ci',95), join=False)\n",
    "    plt.title(f'Usage and Weight on Advice ({condition_name})')\n",
    "    plt.xlabel('Usage Level')\n",
    "    plt.ylabel('Weight on Advice\\n(95% CI)')\n",
    "    usage_levels = ['None', 'Heard Of', 'Used', 'In Class', 'Answer MC']\n",
    "    x_labels = []\n",
    "    for i, level in enumerate(usage_levels):\n",
    "        participant_count = len(advisor_df[advisor_df['usage_level'] == i]['participant_id'].unique())\n",
    "        x_labels.append(f\"{i}: {level}\\n({participant_count})\")\n",
    "    plt.gca().set_xticklabels(x_labels)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f\"../figures/woa_by_advisor_{advisor}.png\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advice Confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.histplot(data=question_df, x=\"advice_confidence\", hue=\"advice_is_correct\", multiple=\"stack\")\n",
    "plt.xlabel(\"Advice Confidence\")\n",
    "plt.legend(['Correct', 'Incorrect'], title='Actual Correctness')\n",
    "plt.title('Advice Confidence and Correctness')\n",
    "plt.savefig('../figures/advice_correctness_belief.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advice Accuracy Belief\n",
    "\n",
    "Calculate expected belief in the accuracy of advice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_prior = (0.5, 0.5)\n",
    "def beta_posterior(row, prior=default_prior):\n",
    "    \"\"\"Calculate the posterior beta distribution for a given row\"\"\"\n",
    "    beta = prior[0] + row['correct_advice_count']\n",
    "    alpha = prior[1] + row['incorrect_advice_count']\n",
    "    return beta / (beta + alpha)\n",
    "question_df['advice_accuracy_belief'] = question_df.apply(beta_posterior, axis=1)\n",
    "\n",
    "print(f\"Mean advice accuracy belief: {question_df['advice_accuracy_belief'].mean():.1%}\")\n",
    "print(f\"Standard deviation of advice accuracy belief: {question_df['advice_accuracy_belief'].std():.1%}\")\n",
    "question_df[\"before_question_10\"] = question_df[\"question_num\"] <= 10\n",
    "sns.histplot(data=question_df, x=\"advice_accuracy_belief\", hue=\"before_question_10\", multiple=\"stack\")\n",
    "plt.title('Distribution of Beliefs')\n",
    "plt.legend(['Question 10 and Before', 'Question 11 and After'], title='Question Number')\n",
    "plt.xlabel('Advice Accuracy Belief')\n",
    "plt.ylabel('Count')\n",
    "plt.savefig('../figures/advice_accuracy_belief.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weight on Advice"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regressions from the section `Weight on Advice`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "regression_results = {}\n",
    "question_df[\"group\"] = 1 # same group for all, necessary for mixed effects\n",
    "default_vcf = {\"question_id\": \"0 + C(question_id)\",\n",
    "               \"participant_id\": \"0 + C(participant_id)\"}\n",
    "\n",
    "def eval_spec(formula:str, vcf:dict=default_vcf, spec_name:str=\"\"):\n",
    "    if spec_name: print(spec_name)\n",
    "    model = sm.MixedLM.from_formula(formula, groups=\"group\",                                                    \n",
    "                                vc_formula=vcf, data=question_df)                                                   \n",
    "    result = model.fit()\n",
    "    return result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression A\n",
    "\n",
    "Regressing on the advisor identity and justification with interaction term and no controls."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula = \"weight_on_advice ~ advisor * give_justification\"\n",
    "result = eval_spec(formula)\n",
    "regression_results[\"A\"] = result\n",
    "result.summary()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression B\n",
    "\n",
    "Regressions on topic familiarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "    + topic_familiarity\"\n",
    "result = eval_spec(formula)\n",
    "regression_results[\"B\"] = result\n",
    "result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B2\n",
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "       + topic_familiarity\"\n",
    "regression_results[\"B2\"] = eval_spec(formula=formula)\n",
    "\n",
    "# B3\n",
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "       + topic_familiarity * advisor\"\n",
    "vcf = {\"question_id\": \"0 + C(question_id)\",\n",
    "       \"participant_id\": \"0 + C(participant_id)\",\n",
    "       \"topic\": \"0 + C(topic)\"}\n",
    "regression_results[\"B3\"] = eval_spec(formula=formula, vcf=vcf)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression C\n",
    "\n",
    "Regress prior usage of AI chatbots."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "    + topic_familiarity + usage_level * advisor\"\n",
    "result = eval_spec(formula)\n",
    "regression_results[\"C\"] = result\n",
    "result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formulas = [\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + usage_level\", # C2\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + C(usage_level)\", # C3\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + used * advisor\", # C4\n",
    "]\n",
    "for i, formula in enumerate(formulas):\n",
    "    regression_results[f\"C{i+2}\"] = eval_spec(formula)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression D\n",
    "\n",
    "Regress on advice quality, measured by whether the advice is correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "    + topic_familiarity + usage_level * \\\n",
    "    advisor + advice_is_correct\"\n",
    "result = eval_spec(formula)\n",
    "regression_results[\"D\"] = result\n",
    "result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formulas = [\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + usage_level\\\n",
    "        * advisor + advice_is_correct + init_advice_confidence\",\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + usage_level\\\n",
    "        * advisor + advice_is_correct\\\n",
    "        + init_advice_confidence + I(init_advice_confidence**2)\",\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + usage_level\\\n",
    "        * advisor + advice_is_correct * give_justification\\\n",
    "        + init_advice_confidence + I(init_advice_confidence**2)\",\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "        + topic_familiarity + usage_level\\\n",
    "        * advisor + advice_is_correct * give_justification\\\n",
    "        + init_advice_confidence + I(init_advice_confidence**2)\\\n",
    "        + advice_is_correct * adjusted_time\"\n",
    "]\n",
    "for i, formula in enumerate(formulas):\n",
    "    regression_results[f\"D{i+2}\"] = eval_spec(formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coef = regression_results['D5'].params['advice_is_correct:adjusted_time']\n",
    "conf_int = regression_results['D5'].conf_int().loc['advice_is_correct:adjusted_time']\n",
    "print(\"For every 10 additional seconds spent incorporating, weight on advice increases by {:.2%} (95% CI: {:.2%}, {:.2%})\".format(coef * 10, conf_int[0] * 10, conf_int[1] * 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula =  \"adjusted_time ~ give_justification\"\n",
    "result = eval_spec(formula)\n",
    "result.summary()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression E\n",
    "\n",
    "Regress on experiences during the survey, include number of questions and beliefs about correctness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "            + topic_familiarity + usage_level * advisor\\\n",
    "            + advice_is_correct + question_num\\\n",
    "            + advice_accuracy_belief * advisor\"\n",
    "result = eval_spec(formula)\n",
    "regression_results[\"E\"] = result\n",
    "result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "formulas = [\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "      + topic_familiarity + usage_level * advisor\\\n",
    "      + advice_is_correct + last_advice_is_correct * advisor\\\n",
    "      + question_num\",\n",
    "    \"weight_on_advice ~ advisor * give_justification\\\n",
    "      + topic_familiarity + usage_level * advisor\\\n",
    "      + advice_is_correct + last_advice_is_correct * advisor\\\n",
    "      + question_num + advice_accuracy_belief * advisor\",\n",
    "]\n",
    "for i, formula in enumerate(formulas):\n",
    "    regression_results[f\"E{i+2}\"] = eval_spec(formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "priors = [(0.05, 0.05), # E4\n",
    "          (0.2, 0.2),   # E5\n",
    "          (1, 1),       # E6\n",
    "          (0.25, 0.75), # E7\n",
    "          (0.75, 0.25)] # E8\n",
    "reg_numbers = range(4, 9)\n",
    "formula = \"weight_on_advice ~ advisor * give_justification\\\n",
    "            + topic_familiarity + usage_level * advisor\\\n",
    "            + advice_is_correct + question_num\\\n",
    "            + advice_accuracy_belief * advisor\"\n",
    "for prior, reg_num in zip(priors, reg_numbers):\n",
    "    question_df['advice_accuracy_belief'] = question_df.apply(beta_posterior, prior=prior, axis=1)\n",
    "    regression_results[f\"E{reg_num}\"] = eval_spec(formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = regression_results[\"E\"]\n",
    "result.conf_int().loc['question_num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standard error on the expert accuracy belief term\n",
    "col_1 = \"advice_accuracy_belief:advisor[T.expert]\"\n",
    "col_2 = \"advice_accuracy_belief\"\n",
    "var_1 = result.cov_params()[col_1][col_1]\n",
    "var_2 = result.cov_params()[col_2][col_2]\n",
    "cov = result.cov_params()[col_1][col_2]\n",
    "se = np.sqrt(var_1 + var_2 + 2 * cov)\n",
    "print(f\"SE: {se}\")\n",
    "# test for expert\n",
    "coef = result.params[\"advice_accuracy_belief\"] + result.params[\"advice_accuracy_belief:advisor[T.expert]\"]\n",
    "# 95% ci\n",
    "print(coef)\n",
    "print(f\"95% CI: {coef - 1.96 * se}, {coef + 1.96 * se}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regressions `E2` and `E3`. Adding a term for the last advice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = regression_results[\"E2\"]\n",
    "result.conf_int().loc['last_advice_is_correct']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standard error on the expert accuracy belief term\n",
    "col_1 = \"last_advice_is_correct:advisor[T.expert]\"\n",
    "col_2 = \"last_advice_is_correct\"\n",
    "var_1 = result.cov_params()[col_1][col_1]\n",
    "var_2 = result.cov_params()[col_2][col_2]\n",
    "cov = result.cov_params()[col_1][col_2]\n",
    "se = np.sqrt(var_1 + var_2 + 2 * cov)\n",
    "print(f\"SE: {se}\")\n",
    "# test for expert\n",
    "coef = result.params[\"last_advice_is_correct\"] + result.params[\"last_advice_is_correct:advisor[T.expert]\"]\n",
    "# 95% ci\n",
    "print(coef)\n",
    "print(f\"95% CI: {coef - 1.96 * se}, {coef + 1.96 * se}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = regression_results[\"E3\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.conf_int().loc['last_advice_is_correct']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standard error on the expert accuracy belief term\n",
    "col_1 = \"last_advice_is_correct:advisor[T.expert]\"\n",
    "col_2 = \"last_advice_is_correct\"\n",
    "var_1 = result.cov_params()[col_1][col_1]\n",
    "var_2 = result.cov_params()[col_2][col_2]\n",
    "cov = result.cov_params()[col_1][col_2]\n",
    "se = np.sqrt(var_1 + var_2 + 2 * cov)\n",
    "print(f\"SE: {se}\")\n",
    "# test for expert\n",
    "coef = result.params[\"last_advice_is_correct\"] + result.params[\"last_advice_is_correct:advisor[T.expert]\"]\n",
    "# 95% ci\n",
    "print(coef)\n",
    "print(f\"95% CI: {coef - 1.96 * se}, {coef + 1.96 * se}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.conf_int().loc['last_advice_is_correct']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename2modelnames = {\n",
    "    \"regression_results\":[\"A\", \"B\", \"C\", \"D\", \"E\"],\n",
    "    \"familiarity_regression_results\":[\"B\", \"B2\", \"B3\"],\n",
    "    \"past_usage_regression_results\":[\"C\"] + [f\"C{i}\" for i in range(2, 5)],\n",
    "    \"correctness_regression_results\":[\"D\"] + [f\"D{i}\" for i in range(2, 6)],\n",
    "    \"experience_regression_results\":[\"E\"] + [f\"E{i}\" for i in range(2, 9)],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename, modelnames in filename2modelnames.items():\n",
    "    summary = summary_col(\n",
    "        [regression_results[model_name] for model_name in modelnames],\n",
    "        stars=True,\n",
    "        float_format='%0.3f',\n",
    "        regressor_order=modelnames,\n",
    "        model_names=modelnames)\n",
    "    latex = summary.as_latex()\n",
    "    # write latex\n",
    "    with open(f\"../tables/{filename}.tex\", 'w') as f:\n",
    "        f.write(latex)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advice Confidence\n",
    "\n",
    "Create callibration curves for all participant answers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_confidence(row: pd.Series, answer_type: str) -> None:\n",
    "    \"\"\"Adds ground truth and advice confidence to `y_prob` and `y_true` lists.\"\"\"\n",
    "    global y_prob, y_true\n",
    "    assert answer_type in ['adjusted', 'init']\n",
    "    for letter in 'ABCD':\n",
    "        y_prob.append(row[f\"{answer_type}_choice{letter}\"])\n",
    "        y_true.append(letter == row['correct_answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([0, 1], [0, 1], linestyle = '--', label = 'Ideally Calibrated', color=\"grey\")\n",
    "\n",
    "y_true = []\n",
    "y_prob = []\n",
    "question_df.apply(extract_confidence, answer_type='init', axis=1)\n",
    "\n",
    "prob_true, prob_pred = calibration_curve(y_true, y_prob, n_bins=10, strategy='uniform')\n",
    "\n",
    "color = \"orange\"\n",
    "\n",
    "# add error bars\n",
    "prob_true_err = []\n",
    "prob_pred_err = []\n",
    "bucket_sizes = []\n",
    "for i in range(len(prob_true)):\n",
    "    bucket_size = len(question_df[(question_df['advice_confidence'] >= prob_pred[i]) & (question_df['advice_confidence'] < prob_pred[i] + 0.1)])\n",
    "    prob_true_err.append(1.96*np.sqrt(prob_true[i] * (1 - prob_true[i]) / bucket_size))\n",
    "    prob_pred_err.append(1.96*np.sqrt(prob_pred[i] * (1 - prob_pred[i]) / bucket_size))\n",
    "    bucket_sizes.append(bucket_size)\n",
    "\n",
    "# calculate ece\n",
    "ece = 0\n",
    "for i in range(len(prob_true)):\n",
    "    ece += bucket_sizes[i] * np.abs(prob_true[i] - prob_pred[i])\n",
    "ece /= len(question_df)\n",
    "print(f\"ECE initial: {ece}\")\n",
    "\n",
    "plt.plot(prob_pred, prob_true, marker = '.', color=color, label=\"Initial\")\n",
    "plt.errorbar(prob_pred, prob_true, yerr = prob_true_err, color = color, linestyle = '', marker = '.', alpha=.5)\n",
    "y_true = []\n",
    "y_prob = []\n",
    "question_df.apply(extract_confidence, answer_type='adjusted', axis=1)\n",
    "\n",
    "prob_true, prob_pred = calibration_curve(y_true, y_prob, n_bins=10, strategy='uniform')\n",
    "\n",
    "color = \"blue\"\n",
    "\n",
    "# add error bars\n",
    "prob_true_err = []\n",
    "prob_pred_err = []\n",
    "bucket_sizes = []\n",
    "for i in range(len(prob_true)):\n",
    "    bucket_size = len(question_df[(question_df['advice_confidence'] >= prob_pred[i]) & (question_df['advice_confidence'] < prob_pred[i] + 0.1)])\n",
    "    prob_true_err.append(1.96*np.sqrt(prob_true[i] * (1 - prob_true[i]) / bucket_size))\n",
    "    prob_pred_err.append(1.96*np.sqrt(prob_pred[i] * (1 - prob_pred[i]) / bucket_size))\n",
    "    bucket_sizes.append(bucket_size)\n",
    "\n",
    "# calculate ece\n",
    "ece = 0\n",
    "for i in range(len(prob_true)):\n",
    "    ece += bucket_sizes[i] * np.abs(prob_true[i] - prob_pred[i])\n",
    "ece /= len(question_df)\n",
    "print(f\"ECE adjusted: {ece}\")\n",
    "\n",
    "plt.plot(prob_pred, prob_true, marker = '.', color=color, label=\"Adjusted\")\n",
    "plt.errorbar(prob_pred, prob_true, yerr = prob_true_err, color = color, linestyle = '', marker = '.', alpha=.5)\n",
    "\n",
    "plt.legend()\n",
    "\n",
    "plt.xlabel('Average Predicted Probability in each bin')\n",
    "plt.ylabel('Ratio of positives\\n(95% CI)')\n",
    "plt.title(\"Calibration of Participant Answers\")\n",
    "plt.savefig('../figures/calibration_curve_participant.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calibration curves on advice only."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = question_df['advice_is_correct']\n",
    "y_prob = question_df['advice_confidence']\n",
    "\n",
    "plt.plot([0, 1], [0, 1], linestyle = '--', label = 'Ideally Calibrated', color=\"grey\")\n",
    "\n",
    "prob_true, prob_pred = calibration_curve(y_true, y_prob, n_bins=10, strategy='uniform')\n",
    "\n",
    "color = \"black\"\n",
    "\n",
    "# add error bars\n",
    "prob_true_err = []\n",
    "prob_pred_err = []\n",
    "bucket_sizes = []\n",
    "for i in range(len(prob_true)):\n",
    "    bucket_size = len(question_df[(question_df['advice_confidence'] >= prob_pred[i]) & (question_df['advice_confidence'] < prob_pred[i] + 0.1)])\n",
    "    prob_true_err.append(1.96*np.sqrt(prob_true[i] * (1 - prob_true[i]) / bucket_size))\n",
    "    prob_pred_err.append(1.96*np.sqrt(prob_pred[i] * (1 - prob_pred[i]) / bucket_size))\n",
    "    bucket_sizes.append(bucket_size)\n",
    "\n",
    "# calcualte ece\n",
    "ece = 0\n",
    "for i in range(len(prob_true)):\n",
    "    ece += bucket_sizes[i] * np.abs(prob_true[i] - prob_pred[i])\n",
    "ece /= len(question_df)\n",
    "print(f\"ECE advice: {ece}\")\n",
    "\n",
    "plt.plot(prob_pred, prob_true, marker = '.', color=color, label='combined')\n",
    "plt.errorbar(prob_pred, prob_true, yerr = prob_true_err, color = color, linestyle = '', marker = '.', alpha=0.3)\n",
    "\n",
    "for advisor, color in zip(['AI chatbot', 'expert'], ['red', 'green']):\n",
    "    sub_df = question_df[question_df['advisor'] == advisor]\n",
    "    y_true = sub_df['advice_is_correct'].astype(float)\n",
    "    y_prob = sub_df['advice_confidence']\n",
    "\n",
    "    prob_true, prob_pred = calibration_curve(y_true, y_prob, n_bins=10, strategy='uniform')\n",
    "\n",
    "    # add error bars\n",
    "    prob_true_err = []\n",
    "    prob_pred_err = []\n",
    "    for i in range(len(prob_true)):\n",
    "        bucket_size = len(sub_df[(sub_df['advice_confidence'] >= prob_pred[i]) & (sub_df['advice_confidence'] < prob_pred[i] + 0.1)])\n",
    "        prob_true_err.append(1.96*np.sqrt(prob_true[i] * (1 - prob_true[i]) / bucket_size))\n",
    "        prob_pred_err.append(1.96*np.sqrt(prob_pred[i] * (1 - prob_pred[i]) / bucket_size))\n",
    "\n",
    "    plt.plot(prob_pred, prob_true, marker = '.', color=color, label = advisor, alpha = 0.3)\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('Average Predicted Probability in each bin')\n",
    "plt.ylabel('Ratio of positives\\n(95% CI)')\n",
    "plt.title(\"Calibration on Advised Choices\")\n",
    "plt.savefig('../figures/calibration_curve_advice.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Actual correctness for the 0% to 10% bucket of confidence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "high = prob_true[0] + prob_true_err[0]\n",
    "low = prob_true[0] - prob_true_err[0]\n",
    "print(f\"{prob_true[0]:.1%} 95% CI: [{low:.1%}, {high:.1%}]\")\n",
    "# calculate expected calibration error\n",
    "print(np.average(np.abs(prob_true - prob_pred), weights=bucket_sizes))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Estimate ECE for each question group and bootstrap a confidence interval."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_ece(df):\n",
    "    prob_true, prob_pred = calibration_curve(df['advice_is_correct'].astype(float), df['advice_confidence'], n_bins=10, strategy='uniform')\n",
    "    bucket_sizes = []\n",
    "    for i in range(len(prob_true)):\n",
    "        bucket_size = len(df[(df['advice_confidence'] >= prob_pred[i]) & (df['advice_confidence'] < prob_pred[i] + 0.1)])\n",
    "        bucket_sizes.append(bucket_size)\n",
    "    return np.average(np.abs(prob_true - prob_pred), weights=bucket_sizes)\n",
    "\n",
    "RESAMPLES = 1000\n",
    "CI = 0.95\n",
    "\n",
    "# calculate ece by question group\n",
    "ece_data = []\n",
    "for question_group, group_df in question_df.groupby('question_group'):\n",
    "    ece_obs = {'question_group':question_group}\n",
    "    ece_obs['ece'] = calculate_ece(group_df)\n",
    "\n",
    "    # bootstrap 95% confidence interval\n",
    "    estimates = []\n",
    "    for i in range(RESAMPLES):\n",
    "        sample = group_df.sample(frac=1, replace=True)\n",
    "        estimates.append(calculate_ece(sample))\n",
    "    estimates = np.array(estimates)\n",
    "    ece_obs['ece_ci'] = (np.quantile(estimates, (1-CI)/2), np.quantile(estimates, CI + (1-CI)/2))\n",
    "    ece_data.append(ece_obs)\n",
    "\n",
    "ece_df = pd.DataFrame(ece_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot barplot\n",
    "plt.bar([f'{i}-{i+4}' for i in range(1, 41, 5)],\n",
    "        ece_df['ece'],\n",
    "        yerr=[ece_df['ece'] - ece_df['ece_ci'].apply(lambda x: x[0]),\n",
    "              ece_df['ece_ci'].apply(lambda x: x[1]) - ece_df['ece']],\n",
    "              capsize=5,\n",
    "        ecolor='black')\n",
    "plt.xlabel('Question Group')\n",
    "plt.ylabel('Expected Calibration Error\\n(95% CI)')\n",
    "plt.title('ECE by Question Group')\n",
    "# add data labels\n",
    "for i, v in enumerate(ece_df['ece']):\n",
    "    plt.text(i - 0.25, 0.01, f\"{v:.2f}\", color='white')\n",
    "plt.savefig('../figures/ece_by_question_group.png')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From here on, use just `chatbot_df`, which are responses for people told it is ChatGPT. Analysis by topic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chatbot_df = question_df[question_df['advisor'] == 'AI chatbot']\n",
    "\n",
    "plt.figure(figsize=(7, 5))\n",
    " # list of topics by mean weight on advice\n",
    "sorted_topics = chatbot_df.groupby('topic')['weight_on_advice'].mean().sort_values().index\n",
    "# color by woa\n",
    "ax = sns.pointplot(data=chatbot_df, x=\"weight_on_advice\", y=\"topic\", join=False, seed=0, orient='h', order = sorted_topics, errorbar=('ci',95), palette='viridis')\n",
    "topic_labels = list(map(lambda t: config.topic2display[t._text], ax.get_yticklabels()))\n",
    "ax.set_yticklabels(topic_labels)\n",
    "\n",
    "\n",
    "average_woa = chatbot_df['weight_on_advice'].mean()\n",
    "plt.axvline(average_woa, color='grey', linestyle='dotted', label='Average Weight on Advice')\n",
    "\n",
    "plt.text(average_woa + 0.01, 0.5, str(round(average_woa, 3)), color='grey', verticalalignment='center')\n",
    "\n",
    "plt.xlabel('Weight on Advice\\n(95% CI)')\n",
    "plt.ylabel('Topic')\n",
    "plt.title('Weight on Advice by Topic')\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"../figures/woa_by_topic.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chatbot_df['supercategory'] = chatbot_df['topic'].apply(config.topic2category.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = sns.pointplot(data=chatbot_df, x=\"weight_on_advice\", y=\"supercategory\", join=False, seed=0, orient='h', errorbar=('ci',95))\n",
    "topic_labels = list(map(lambda t: config.category2display[t._text], ax.get_yticklabels()))\n",
    "ax.set_yticklabels(topic_labels)\n",
    "plt.xlabel('Weight on Advice')\n",
    "plt.ylabel('Topic')\n",
    "plt.title('Weight on Advice by Category')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "woa_and_correctness_df = chatbot_df.groupby('topic')[['weight_on_advice', 'advice_confidence']].agg(['mean','count'])\n",
    "# scatterplot\n",
    "plt.scatter(woa_and_correctness_df['weight_on_advice']['mean'],\n",
    "            woa_and_correctness_df['advice_confidence']['mean'],\n",
    "            s=woa_and_correctness_df['advice_confidence']['count'], \n",
    "            alpha=.2)\n",
    "# annotate\n",
    "keep_topics = ['elementary_mathematics','high_school_microeconomics','high_school_geography','human_sexuality','nutrition','high_school_government_and_politics', 'high_school_chemistry', 'high_school_biology']\n",
    "\n",
    "for i, topic in enumerate(woa_and_correctness_df.index):\n",
    "    if topic not in keep_topics:\n",
    "        continue\n",
    "    x = woa_and_correctness_df['weight_on_advice']['mean'][i]\n",
    "    y = woa_and_correctness_df['advice_confidence']['mean'][i]\n",
    "    display_text = \" \" + config.topic2display[topic]\n",
    "    plt.annotate(display_text, (x, y), fontsize=8)\n",
    "\n",
    "plt.xlabel('Average Weight on Advice')\n",
    "plt.ylabel('Average Advice Confidence')\n",
    "plt.title('Weight on Advice and Advice Confidence by Topic')\n",
    "plt.tight_layout()\n",
    "plt.savefig('../figures/woa_and_correctness.png')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Correlation between weight on advice and advice confidence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pearsonr(chatbot_df['weight_on_advice'], chatbot_df['advice_confidence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_result_df = pd.read_csv('../data/model_output/results_0425.csv')\n",
    "model_result_df = model_result_df[model_result_df['topic'].isin(config.topics)]\n",
    "model_result_df['answer_correct'] = model_result_df['answer'] == model_result_df['correct_answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conduct t-tests by topic\n",
    "topic2significance = {}\n",
    "all_correctness, all_beliefs = [], []\n",
    "for topic in config.topics:\n",
    "    correctness = model_result_df[model_result_df['topic'] == topic]['answer_correct']\n",
    "    beliefs = chatbot_df[chatbot_df['topic'] == topic]['advice_confidence']\n",
    "    t, p = brunnermunzel(correctness, beliefs)\n",
    "    topic2significance[topic] = p\n",
    "    all_correctness.append(correctness.mean())\n",
    "    all_beliefs.append(beliefs.mean())\n",
    "\n",
    "# FDR correction\n",
    "rejections = fdrcorrection(list(topic2significance.values()))\n",
    "topic2reject = dict(zip(topic2significance.keys(), rejections[0]))\n",
    "pearsonr(all_correctness, all_beliefs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SIG_LEVEL = 0.05\n",
    "plt.figure(figsize=(10, 6))\n",
    "sorted_topics = model_result_df.groupby('topic')['answer_correct'].mean().sort_values().index\n",
    "ax1 = sns.pointplot(data=chatbot_df, x=\"advice_confidence\", y=\"topic\", label='Advice Confidence}', errorbar=('ci', 95), join=False, orient='h', order = sorted_topics)\n",
    "ax2 = sns.pointplot(data=model_result_df, x=\"answer_correct\", y=\"topic\", label='Advice Accuracy', color='red', errorbar=('ci', 95), join=False, orient='h', order = sorted_topics)\n",
    "topic_labels = list(map(lambda t: config.topic2display[t._text], ax1.get_yticklabels()))\n",
    "\n",
    "ax1.set_yticklabels(topic_labels)\n",
    "\n",
    "display2topic = {v:k for k,v in config.topic2display.items()}\n",
    "\n",
    "for tick in ax1.get_yticklabels():\n",
    "    topic = display2topic[tick._text]\n",
    "    if topic2reject[topic]:\n",
    "        tick.set_fontweight('bold')\n",
    "    else:\n",
    "        tick.set_fontweight('normal')\n",
    "\n",
    "for line in ax1.lines:\n",
    "    line.set_alpha(0.3)\n",
    "for line in ax2.lines:\n",
    "    line.set_alpha(0.3)\n",
    "\n",
    "plt.subplots_adjust(left=0.35)\n",
    "plt.xlabel('Probability\\n(95% CI)')\n",
    "plt.ylabel(f\"Topic\")\n",
    "plt.title('Misperception in Advice Accuracy')\n",
    "plt.legend()\n",
    "\n",
    "# set sigdif topics to be bold\n",
    "ax = plt.gca()\n",
    "plt.savefig('../figures/misperception_in_advice_accuracy.png')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating optimal weight on advice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def brier_score(row: pd.Series, type: str) -> float:\n",
    "    \"\"\"Calculate the Brier score for a single row.\"\"\"\n",
    "    assert type in ['init', 'adjusted', 'simulated']\n",
    "    return sum((row[f\"{type}_choice{letter_choice}\"] - int(letter_choice == row['correct_answer'])) ** 2 for letter_choice in \"ABCD\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def take_advice(row: pd.Series, weight_on_advice_logg: float):\n",
    "    \"\"\"Simulate taking advice from the AI chatbot.\"\"\"\n",
    "    assert 0 <= weight_on_advice_logg <= 1\n",
    "    \n",
    "    simulated_choices = []\n",
    "    advice_choice = row[f\"advice_answer\"]\n",
    "\n",
    "    if advice_choice not in \"ABCD\":\n",
    "        return pd.Series(list(row[[f\"init_choice{letter_choice}\" for letter_choice in \"ABCD\"]]))\n",
    "\n",
    "    advice_gap = 1 - row[f\"init_choice{advice_choice}\"]\n",
    "    scale_factor = advice_gap * weight_on_advice_logg\n",
    "    \n",
    "    for letter_choice in \"ABCD\":\n",
    "        weight = row[f\"init_choice{letter_choice}\"] * (1 - scale_factor)\n",
    "        if letter_choice == advice_choice:\n",
    "            weight += scale_factor\n",
    "        simulated_choices.append(weight)\n",
    "\n",
    "    return pd.Series(simulated_choices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WOAs = np.linspace(0, 1, 101)\n",
    "brier_scores = []\n",
    "for woa in tqdm(WOAs):\n",
    "    simulated_df = chatbot_df.apply(take_advice, weight_on_advice_logg=woa, axis=1)\n",
    "    simulated_df.columns = [f\"simulated_choice{letter_choice}\" for letter_choice in \"ABCD\"]\n",
    "    chatbot_df[simulated_df.columns] = simulated_df\n",
    "    chatbot_df['simulated_answer'] =  chatbot_df.apply(pick_answer, axis=1, type='simulated')\n",
    "    chatbot_df['simulated_answer_correct'] = chatbot_df['simulated_answer'] == chatbot_df['correct_answer']\n",
    "    chatbot_df['simulated_brier_score'] = chatbot_df.apply(brier_score, type='simulated', axis=1)\n",
    "    brier_scores.append(chatbot_df['simulated_brier_score'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adjusted brier score\n",
    "chatbot_df['adjusted_brier_score'] = chatbot_df.apply(brier_score, type='adjusted', axis=1)\n",
    "average_bs = chatbot_df['adjusted_brier_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimal_woa = WOAs[np.argmin(brier_scores)]\n",
    "print(f\"Optimal WOA: {optimal_woa:.3f}\\nOptiaml Brier score: {min(brier_scores):.3f}\")\n",
    "average_woa = chatbot_df['weight_on_advice'].mean()\n",
    "print(f\"Average WOA: {average_woa:.3f}\\nAverage Brier score: {average_bs:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# closest woa to average woa\n",
    "closest_index = np.argmin(np.abs(WOAs - average_woa))\n",
    "closest_woa = WOAs[closest_index]\n",
    "print(f\"Closest WOA to average WOA: {closest_woa:.3f}, Brier score: {brier_scores[closest_index]:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 5))\n",
    "plt.plot(WOAs, brier_scores, label='Simulated Brier Score')\n",
    "plt.xlabel('Weight on Advice')\n",
    "plt.ylabel('Brier Score')\n",
    "plt.vlines(optimal_woa, min(brier_scores) - .05, max(brier_scores) + .05, linestyles='dashed', colors='red', label='Optimal Weight on Advice')\n",
    "plt.annotate(f\"{optimal_woa:.1%}\", (optimal_woa + .01, min(brier_scores) - .05), fontsize=10, color='red')\n",
    "plt.vlines(average_woa, min(brier_scores) - .05, max(brier_scores) + .05, linestyles='dashed', colors='green', label='Average Weight on Advice')\n",
    "plt.annotate(f\"{average_woa:.1%}\", (average_woa + .01, min(brier_scores) - .05), fontsize=10, color='green')\n",
    "plt.hlines(average_bs, 0, max(WOAs), linestyles='dotted', colors='orange', label='Actual Average Brier Score')\n",
    "plt.annotate(f\"{average_bs:.3f}\", (max(WOAs), average_bs), fontsize=10, color='orange', backgroundcolor='white')\n",
    "plt.scatter(x=[average_woa], y=[average_bs], color='black', label='Actual Average Brier Score')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.set_major_formatter(mtick.PercentFormatter(1))\n",
    "plt.title('Optimal Weight on Advice')\n",
    "plt.legend()\n",
    "plt.savefig('../figures/woa_brier_score.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_df = chatbot_df.copy()\n",
    "def sim_proportionate(row):\n",
    "    new_row = row.copy()\n",
    "    advice_answer = row['advice_answer']\n",
    "    if advice_answer not in \"ABCD\":\n",
    "        return new_row\n",
    "\n",
    "    abs_change = (1 - row[f\"init_choice{advice_answer}\"]) * row['weight_on_advice']\n",
    "    \n",
    "    new_row['simulated_choiceA'] = row['init_choiceA'] * (1 - abs_change)\n",
    "    new_row['simulated_choiceB'] = row['init_choiceB'] * (1 - abs_change)\n",
    "    new_row['simulated_choiceC'] = row['init_choiceC'] * (1 - abs_change)\n",
    "    new_row['simulated_choiceD'] = row['init_choiceD'] * (1 - abs_change)\n",
    "    new_row[f\"simulated_choice{row['advice_answer']}\"] += abs_change\n",
    "    return new_row\n",
    "sim_df = sim_df.apply(sim_proportionate, axis=1)\n",
    "sim_df['brier_score'] = sim_df.apply(brier_score, type='simulated', axis=1)\n",
    "sim_df['brier_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_woa = chatbot_df['weight_on_advice'].mean()\n",
    "sim_df = chatbot_df.copy()\n",
    "def sim_depolarize(row, scale=.5):\n",
    "    \"\"\"\n",
    "    Simulate a shrinking of the weight on advice towards average but\n",
    "    proportionately maintaining other adjusted answers.\n",
    "    \"\"\"\n",
    "    new_row = row.copy()\n",
    "\n",
    "    advice_answer = row['advice_answer'] \n",
    "    if advice_answer not in \"ABCD\":\n",
    "        for letter in \"ABCD\":\n",
    "            new_row[f\"simulated_choice{letter}\"] = row[f\"init_choice{letter}\"]\n",
    "        return new_row\n",
    "\n",
    "    advice_weight = row[f\"adjusted_choice{advice_answer}\"]\n",
    "    woa = row['weight_on_advice'] * (1 - row[f\"init_choice{advice_answer}\"])\n",
    "    new_woa = avg_woa * (1 - scale) + woa * scale\n",
    "    new_row['weight_on_advice'] = new_woa\n",
    "    new_advice_weight = new_woa + row[f\"init_choice{advice_answer}\"]\n",
    "\n",
    "    if advice_weight != 1:\n",
    "        for letter in \"ABCD\":\n",
    "            new_row[f\"simulated_choice{letter}\"] = row[f\"adjusted_choice{letter}\"]  / (1-advice_weight) * (1-new_advice_weight)\n",
    "    else:\n",
    "        for letter in \"ABCD\":\n",
    "            new_row[f\"simulated_choice{letter}\"] = (1-new_advice_weight) / 3\n",
    "    new_row[f\"simulated_choice{advice_answer}\"] = new_advice_weight\n",
    "\n",
    "    return new_row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ss = np.linspace(0, 1, 101)\n",
    "brier_scores = []\n",
    "for s in tqdm(Ss):\n",
    "    sim_df = sim_df.apply(sim_depolarize, axis=1, scale=s)\n",
    "    sim_df['brier_score'] = sim_df.apply(brier_score, type='simulated', axis=1)\n",
    "    bs = sim_df['brier_score'].mean()\n",
    "    brier_scores.append(bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate optimal s\n",
    "optimal_s = Ss[np.argmin(brier_scores)]\n",
    "print(f\"Optimal s: {optimal_s:.3f}\\nOptiaml Brier score: {min(brier_scores):.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot scaling\n",
    "plt.figure(figsize=(6, 4))\n",
    "plt.plot(Ss, brier_scores, label='Simulated Brier Score')\n",
    "plt.xlabel('Shrinking')\n",
    "plt.ylabel('Brier Score')\n",
    "plt.vlines(optimal_s, min(brier_scores) - .05, max(brier_scores) + .05, linestyles='dashed', colors='blue', label='Optimal Scaling')\n",
    "plt.annotate(f\"{optimal_s:.1%}\", (optimal_s + .01, min(brier_scores) - .05), fontsize=10, color='blue')\n",
    "plt.title('Optimal Shrinking')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.set_major_formatter(mtick.PercentFormatter(1))\n",
    "plt.savefig('../figures/shrinking_brier_score.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ss = np.linspace(0, 1, 101)\n",
    "brier_scores = []\n",
    "for s in tqdm(Ss):\n",
    "    sim_df = sim_df.apply(sim_depolarize, axis=1, scale=s).apply(sim_proportionate, axis=1)\n",
    "    sim_df['brier_score'] = sim_df.apply(brier_score, type='simulated', axis=1)\n",
    "    bs = sim_df['brier_score'].mean()\n",
    "    brier_scores.append(bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate optimal s\n",
    "optimal_s = Ss[np.argmin(brier_scores)]\n",
    "print(f\"Optimal s: {optimal_s:.3f}\\nOptiaml Brier score: {min(brier_scores):.3f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Miscellaneous"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization of Usages"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize open-ended responses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "usages = list(map(str, participant_df[\"usage_description\"].dropna()))\n",
    "# word cloud with bigrams\n",
    "wordcloud = WordCloud(width=2000,\n",
    "                      height=1000,\n",
    "                      background_color=\"white\",\n",
    "                      max_words=300,\n",
    "                      contour_width=3,\n",
    "                      contour_color='steelblue',\n",
    "                      collocations=True,\n",
    "                      random_state=3)\n",
    "wordcloud.generate(\" \".join(usages))\n",
    "plt.figure(figsize=(20, 10))\n",
    "plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
    "plt.axis(\"off\")\n",
    "plt.savefig('../figures/usage_wordcloud.png')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Power Analysis\n",
    "\n",
    "For conclusion section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "participant_df = question_df.groupby('participant_id').agg({\n",
    "    'weight_on_advice':'mean',\n",
    "    'advisor':'first',\n",
    "    'give_justification':'first',\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.stats.power as smp\n",
    "\n",
    "power_analysis = smp.TTestIndPower()\n",
    "effect_size = power_analysis.solve_power(nobs1=len(participant_df), power=0.8, alpha=0.05)\n",
    "effect_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate cohen's d\n",
    "def cohen_d(x, y):\n",
    "    nx = len(x)\n",
    "    ny = len(y)\n",
    "    dof = nx + ny - 2\n",
    "    return (np.mean(x) - np.mean(y)) / np.sqrt(((nx - 1) * np.std(x, ddof=1) ** 2 + (ny - 1) * np.std(y, ddof=1) ** 2) / dof)\n",
    "x = participant_df[participant_df['advisor'] == 'AI chatbot']['weight_on_advice']\n",
    "y = participant_df[participant_df['advisor'] == 'expert']['weight_on_advice']\n",
    "advisor_effect_size = cohen_d(x, y)\n",
    "print(f\"Advisor effect size: {advisor_effect_size:.3f}\")\n",
    "advisor_sample_size = power_analysis.solve_power(effect_size=advisor_effect_size, power=0.8, alpha=0.05)\n",
    "print(f\"Advisor sample size: {advisor_sample_size:.0f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = participant_df[participant_df['give_justification'] == \"yes\"]['weight_on_advice']\n",
    "y = participant_df[participant_df['give_justification'] == \"no\"]['weight_on_advice']\n",
    "justification_effect_size = cohen_d(x, y)\n",
    "print(f\"Justification effect size: {justification_effect_size:.3f}\")\n",
    "justification_sample_size = power_analysis.solve_power(effect_size=justification_effect_size, power=0.8, alpha=0.05)\n",
    "print(f\"Justification sample size: {justification_sample_size:.0f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}