{
  "@context": {
    "dcterms": "http://purl.org/dc/terms/",
    "dcmitype": "http://purl.org/dc/dcmitype/"
  },
  "dcterms:identifier": [
    "tag:aim-pro.eu,2026:oer/e71a9e5763e5",
    "https://arxiv.org/abs/2510.22581",
    "arxiv:2510.22581"
  ],
  "dcterms:title": [
    "Pedagogy-driven Evaluation of Generative AI-powered Intelligent Tutoring Systems"
  ],
  "dcterms:type": [
    {
      "@id": "dcmitype:Text"
    },
    "preprint"
  ],
  "dcterms:creator": [
    "Kaushal Kumar Maurya",
    "Ekaterina Kochmar"
  ],
  "dcterms:description": [
    "The interdisciplinary research domain of Artificial Intelligence in Education (AIED) has a long history of developing Intelligent Tutoring Systems (ITSs) by integrating insights from technological advancements, educational theories, and cognitive psychology. The remarkable success of generative AI (GenAI) models has accelerated the development of large language model (LLM)-powered ITSs, which have potential to imitate human-like, pedagogically rich, and cognitively demanding tutoring. However, the progress and impact of these systems remain largely untraceable due to the absence of reliable, universally accepted, and pedagogy-driven evaluation frameworks and benchmarks. Most existing educational dialogue-based ITS evaluations rely on subjective protocols and non-standardized benchmarks, leading to inconsistencies and limited generalizability. In this work, we take a step back from mainstream ITS development and provide comprehensive state-of-the-art evaluation practices, highlighting associated challenges through real-world case studies from careful and caring AIED research. Finally, building on insights from previous interdisciplinary AIED research, we propose three practical, feasible, and theoretically grounded research directions, rooted in learning science principles and aimed at establishing fair, unified, and scalable evaluation methodologies for ITSs."
  ],
  "dcterms:subject": [
    "cs.CL (arxiv)"
  ],
  "dcterms:language": [
    "en"
  ],
  "dcterms:license": [
    {
      "@id": "http://creativecommons.org/licenses/by-nc-sa/4.0/"
    }
  ],
  "dcterms:rights": [
    "CC-BY-NC-SA-4.0 — assessed as OPEN_NC by the harvester's licence gate. Conditions: attribution required, non-commercial use only, adaptations must carry the same licence. open but Non-Commercial — SME/commercial reuse needs care"
  ],
  "dcterms:accessRights": [
    "open"
  ],
  "dcterms:format": [
    "text/markdown"
  ],
  "dcterms:extent": [
    "44988 bytes (extracted text)"
  ],
  "dcterms:issued": [
    "2025-10-26"
  ],
  "dcterms:provenance": [
    "Retrieved from arXiv on 2026-10-09 in response to the search string “(all:\"artificial intelligence\" OR all:\"machine learning\" OR all:\"generative AI\" OR all:\"deep learning\" OR all:\"reinforcement learning\" OR all:\"large language model\") AND (all:\"AI concepts\" OR all:\"types of AI\" OR all:\"AI fundamentals\" OR all:\"recognizing AI\" OR all:\"recognising AI\" OR all:\"general versus narrow AI\" OR all:\"narrow AI\" OR all:\"general AI\" OR all:\"machine intelligence\" OR all:\"AI strengths and weaknesses\" OR all:\"traditional software\" OR all:\"rule-based systems\" OR all:\"introduction to AI\" OR all:\"introduction to artificial intelligence\" OR all:\"artificial intelligence introduction\" OR all:\"AI primer\" OR all:\"foundations of artificial intelligence\" OR all:\"overview of AI\" OR all:\"understanding AI\" OR all:\"history of AI\" OR all:\"AI essentials\" OR all:\"AI terminology\" OR all:\"metaphors for AI\" OR all:\"AI fundamental concepts\" OR all:\"AI key concepts\" OR all:\"philosophy of AI\" OR all:\"critical AI literacy\")”. arXiv served the resource and is not asserted to be its publisher or author.",
    "Text extracted from latexml-html to Markdown by arxiv-html; the original is retained unchanged beside it."
  ],
  "inferred": {
    "dcterms:references": [
      {
        "value": {
          "@id": "https://arxiv.org/abs/2211.11772",
          "title": "arXiv:2211.11772"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 136,
              "basis": "reference",
              "literal": "arXiv:2211.11772"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2106.03873",
          "title": "arXiv:2106.03873"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 137,
              "basis": "reference",
              "literal": "arXiv:2106.03873"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2009.06978",
          "title": "arXiv:2009.06978"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 139,
              "basis": "reference",
              "literal": "arXiv:2009.06978"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2411.15594",
          "title": "arXiv:2411.15594"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 143,
              "basis": "reference",
              "literal": "arXiv:2411.15594"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2407.12687",
          "title": "arXiv:2407.12687"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 148,
              "basis": "reference",
              "literal": "arXiv:2407.12687"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/1609.02907",
          "title": "arXiv:1609.02907"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 150,
              "basis": "reference",
              "literal": "arXiv:1609.02907"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2305.14536",
          "title": "arXiv:2305.14536"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 154,
              "basis": "reference",
              "literal": "arXiv:2305.14536"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2502.18940",
          "title": "arXiv:2502.18940"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 155,
              "basis": "reference",
              "literal": "arXiv:2502.18940"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2406.13903",
          "title": "arXiv:2406.13903"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 168,
              "basis": "reference",
              "literal": "arXiv:2406.13903"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2204.09652",
          "title": "arXiv:2204.09652"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 175,
              "basis": "reference",
              "literal": "arXiv:2204.09652"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2205.07540",
          "title": "arXiv:2205.07540"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 177,
              "basis": "reference",
              "literal": "arXiv:2205.07540"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2412.16429",
          "title": "arXiv:2412.16429"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 178,
              "basis": "reference",
              "literal": "arXiv:2412.16429"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2410.03017",
          "title": "arXiv:2410.03017"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 184,
              "basis": "reference",
              "literal": "arXiv:2410.03017"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/2402.02420",
          "title": "arXiv:2402.02420"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 185,
              "basis": "reference",
              "literal": "arXiv:2402.02420"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "value": {
          "@id": "https://arxiv.org/abs/1904.09675",
          "title": "arXiv:1904.09675"
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 191,
              "basis": "reference",
              "literal": "arXiv:1904.09675"
            }
          ],
          "kind": "citation"
        }
      }
    ]
  }
}