JavaScript なしで自己ペース読みの実験を作る

pip install potato-annotation==2.13.1

spr-study/
├── config.yaml
├── data/list1.jsonl
├── data/list2.jsonl
├── surveys/consent.json
├── surveys/instructions.json
└── surveys/background.json

import json
import random
 
import pandas as pd
 
sap = pd.read_excel("Items for all subsets.xlsx", sheet_name="NPSNPZMVRR")
mvrr = sap[sap.condition == "MVRR_UAMB"]
fillers = [json.loads(line) for line in open("fillers.jsonl")]
practice = [json.loads(line) for line in open("practice.jsonl")]
 
rng = random.Random(20261029)
for n, name in enumerate(["list1", "list2"]):
    rows = []
    for _, r in mvrr.iterrows():
        item = int(r["item"])
        ambiguous = (item % 2 == 1) == (n == 0)  # the Latin square
        sentence = r["ambiguous"] if ambiguous else r["Unnamed: 3"]
        verb = r["disambPositionAmb"] if ambiguous else r["disambPositionUnamb"]
        rows.append({
            "id": f"mvrr{item:02d}_{'amb' if ambiguous else 'unamb'}",
            "item": item,
            "condition": "ambiguous" if ambiguous else "unambiguous",
            "sentence": " ".join(sentence.split()),
            "question": r["Question"].strip(),
            "answer": r["Option1"] if r["Answer"] == 1 else r["Option0"],
            "critical_region": int(verb) - 1,  # SAP counts words from 1
        })
    rows += [{**f, "id": f"{f['id']}_{name}"} for f in fillers]
    while True:  # no two items of one condition in a row
        rng.shuffle(rows)
        if all(a["condition"] == "filler" or a["condition"] != b["condition"]
               for a, b in zip(rows, rows[1:])):
            break
    with open(f"data/{name}.jsonl", "w") as out:
        for row in [{**p, "id": f"{p['id']}_{name}"} for p in practice] + rows:
            out.write(json.dumps(row) + "\n")

{"id": "mvrr01_amb", "item": 1, "condition": "ambiguous", "sentence": "The suspect sent the file deserved further investigation given the new evidence.", "question": "Did the suspect send the file to someone?", "answer": "No", "critical_region": 5}

annotation_task_name: Garden-path self-paced reading
task_dir: .
output_annotation_dir: annotation_output/
data_files: []
item_properties:
  id_key: id
  text_key: sentence
 
login:
  type: url_direct
  url_argument: PROLIFIC_PID
user_config:
  allow_all_users: true
  users: []
 
crowdsourcing:
  provider: prolific
  prolific:
    completion:
      code: C1READING
 
conditions:
  factors:
    list: [list1, list2]
 
assignment_strategy: batch
batch_assignment:
  auto_assign_annotators: true
  group_by_condition: list
  groups:
    - {name: list1, data_file: data/list1.jsonl}
    - {name: list2, data_file: data/list2.jsonl}
 
phases:
  order: [consent, instructions, annotation, background]
  consent:
    type: consent
    file: surveys/consent.json
  instructions:
    type: instructions
    file: surveys/instructions.json
  annotation:
    type: annotation
  background:
    type: poststudy
    file: surveys/background.json
 
trial_timing:
  enabled: true
  fixation_ms: 400
  iti_ms: 500
  presentation:
    mode: moving_window
    field: sentence
    advance_key: space
    question_field: question
  response:
    schema: comprehension
    keys: {f: "Yes", j: "No"}
 
annotation_schemes:
  - annotation_type: radio
    name: comprehension
    description: Answer the question
    labels: ["Yes", "No"]
    label_requirement:
      required: true
 
export_include_phase_data: true

[
  {
    "name": "study_information",
    "annotation_type": "pure_display",
    "description": "About this study",
    "labels": [
      "You will read 38 sentences one word at a time and answer a yes-or-no question after each. The study takes about 15 minutes.",
      "Your key presses and answers are stored on a university server under your Prolific ID.",
      "If you do not want to take part, close this page and return the study on Prolific."
    ]
  },
  {
    "name": "consent",
    "annotation_type": "radio",
    "description": "Do you agree to take part in this study?",
    "labels": ["I agree to take part"],
    "label_requirement": {"required_label": ["I agree to take part"]}
  }
]

[
  {
    "name": "how_to_read",
    "annotation_type": "pure_display",
    "description": "How the reading works",
    "labels": [
      "Each sentence first appears as a row of dashes. Press the space bar to see the first word, and press it again for each next word. Each word disappears when the next one appears.",
      "Read at your normal pace. After the last word, a question appears. Press F for yes and J for no.",
      "The first two sentences are practice."
    ]
  }
]

[
  {
    "name": "native_english",
    "annotation_type": "radio",
    "description": "Is English your native language?",
    "labels": ["Yes", "No"],
    "label_requirement": {"required_label": ["Yes", "No"]}
  },
  {
    "name": "comments",
    "annotation_type": "text",
    "description": "Anything else you want to tell us? (optional)"
  }
]

potato start config.yaml -p 8000

python -m potato.export --config config.yaml --format csv --output results/

user_id,instance_id,condition_list,region,text,rt_ms,correct,side,errors,trial_outcome,onset_epoch_ms
r01,mvrr12_amb,list2,4,the,295.9,,,,response,1793287302649
r01,mvrr12_amb,list2,5,training,458.5,,,,response,1793287302649
r01,mvrr12_amb,list2,6,seemed,553.2,,,,response,1793287302649
r01,mvrr12_amb,list2,7,unnecessarily,564.2,,,,response,1793287302649
r01,mvrr12_amb,list2,8,demanding,433.9,,,,response,1793287302649

import numpy as np
import pandas as pd
import statsmodels.formula.api as smf
 
items = pd.concat([pd.read_json(f"data/{f}.jsonl", lines=True) for f in ("list1", "list2")])
items = items.rename(columns={"id": "instance_id"})
trials = pd.read_csv("results/trials.csv").merge(items, on="instance_id")
regions = pd.read_csv("results/regions.csv").merge(items, on="instance_id")
 
# Comprehension accuracy per participant, practice excluded.
trials = trials[trials.condition != "practice"]
trials["correct"] = trials.label == trials.answer
accuracy = trials.groupby("user_id").correct.mean()
keep = accuracy[accuracy >= 0.8].index
print(accuracy.round(2).to_dict())
 
# Drop practice, low-accuracy readers, reloaded trials, and extreme times.
flagged = trials[trials["flags"].fillna("").str.contains("reloaded_mid_trial")]
rt = regions[(regions.condition != "practice") & regions.user_id.isin(keep)]
rt = rt.merge(flagged[["user_id", "instance_id"]], how="left", indicator=True)
rt = rt[(rt._merge == "left_only") & rt.rt_ms.between(100, 2000)].copy()
 
# Residual reading time: each reader's times regressed on word length.
rt["length"] = rt.text.str.replace(r"[^A-Za-z]", "", regex=True).str.len()
 
 
def residuals(d):
    slope, intercept = np.polyfit(d.length, d.rt_ms, 1)
    return d.rt_ms - (intercept + slope * d.length)
 
 
rt["residual"] = pd.concat([residuals(d) for _, d in rt.groupby("user_id")])
 
# Positions relative to the disambiguating verb, critical items only.
crit = rt[rt.condition.isin(["ambiguous", "unambiguous"])].copy()
crit["position"] = (crit.region - crit.critical_region).astype(int)
window = crit[crit.position.between(-1, 2)]
window.to_csv("window.csv", index=False)
print(window.pivot_table(index="position", columns="condition",
                         values="residual", aggfunc="mean").round(1))
 
# Mixed model at the verb and the two words after it.
spill = window[window.position >= 0]
fit = smf.mixedlm("residual ~ condition", spill, groups="user_id",
                  vc_formula={"item": "0 + C(item)"}).fit()
print(fit.summary().tables[1])

{'r01': 0.92, 'r02': 0.83, 'r03': 0.89, 'r04': 0.97, 'r05': 0.94, 'r06': 0.67, 'r07': 0.92, 'r08': 0.89, 'r09': 0.89, 'r10': 0.94, 'r11': 0.81, 'r12': 0.94}
condition  ambiguous  unambiguous
position
-1               0.8          2.3
 0              80.7         -7.8
 1              56.4        -13.8
 2              18.5        -10.4
                            Coef. Std.Err.        z  P>|z|   [0.025   0.975]
Intercept                  51.865    2.559   20.268  0.000   46.850   56.881
condition[T.unambiguous]  -62.542    3.626  -17.249  0.000  -69.648  -55.435
item Var                    0.000    1.955

library(lme4)
window <- read.csv("window.csv")
spill <- subset(window, position >= 0)
fit <- lmer(residual ~ condition + (1 + condition | user_id) + (1 + condition | item),
            data = spill)
summary(fit)

trial_timing:
  enabled: true
  fixation_ms: 400
  iti_ms: 500
  presentation:
    mode: maze
    field: sentence
    distractor_field: distractors
    maze_keys: {left: e, right: i}
    on_error: redo
 
annotation_schemes:
  - annotation_type: pure_display
    name: maze_instructions
    description: How to read
    labels:
      - Press E for the left word or I for the right word, whichever continues the sentence.

{"id": "mvrr01_amb", "item": 1, "condition": "ambiguous", "sentence": "The suspect sent the file deserved further investigation given the new evidence.", "distractors": "ran very of went ninety whose upon slept seemed of often.", "critical_region": 5}

potato import -f ibex -i my_experiment/ -o my-study/