{
  "identifier": "tag:aim-pro.eu,2026:oer/e71a9e5763e5/record",
  "profile": "aimpro-oer-profile/1",
  "normaliser": "oerlib.metadata/2",
  "status": "ready",
  "normalised_at": "2026-10-09T09:52:40+00:00",
  "resource": {
    "id": "e71a9e5763e5",
    "identifier": "tag:aim-pro.eu,2026:oer/e71a9e5763e5",
    "uri": "https://arxiv.org/abs/2510.22581",
    "title": "Pedagogy-driven Evaluation of Generative AI-powered Intelligent Tutoring Systems",
    "description": "The interdisciplinary research domain of Artificial Intelligence in Education (AIED) has a long history of developing Intelligent Tutoring Systems (ITSs) by integrating insights from technological advancements, educational theories, and cognitive psychology. The remarkable success of generative AI (GenAI) models has accelerated the development of large language model (LLM)-powered ITSs, which have potential to imitate human-like, pedagogically rich, and cognitively demanding tutoring. However, the progress and impact of these systems remain largely untraceable due to the absence of reliable, universally accepted, and pedagogy-driven evaluation frameworks and benchmarks. Most existing educational dialogue-based ITS evaluations rely on subjective protocols and non-standardized benchmarks, leading to inconsistencies and limited generalizability. In this work, we take a step back from mainstream ITS development and provide comprehensive state-of-the-art evaluation practices, highlighting associated challenges through real-world case studies from careful and caring AIED research. Finally, building on insights from previous interdisciplinary AIED research, we propose three practical, feasible, and theoretically grounded research directions, rooted in learning science principles and aimed at establishing fair, unified, and scalable evaluation methodologies for ITSs.",
    "creators": [
      "Kaushal Kumar Maurya",
      "Ekaterina Kochmar"
    ],
    "keywords": [
      "cs.CL"
    ],
    "kind": "html",
    "content_language": "en",
    "declared_language": "en",
    "word_count": 6435,
    "content_hash": "ed3af0749bd57af1",
    "metadata_only": false
  },
  "licence": {
    "raw": "CC-BY-NC-SA-4.0",
    "spdx": "CC-BY-NC-SA-4.0",
    "uri": "http://creativecommons.org/licenses/by-nc-sa/4.0/",
    "verdict": "OPEN_NC",
    "note": "open but Non-Commercial — SME/commercial reuse needs care",
    "ingestable": true,
    "commercial_ok": false,
    "derivable": true,
    "share_alike": true,
    "evidence": {
      "scope": "this submitted version of the paper",
      "method": "arXiv OAI-PMH GetRecord (metadataPrefix=arXiv)",
      "schema": 1,
      "status": "captured",
      "location": "arXiv:arXiv/arXiv:license",
      "source_url": "https://oaipmh.arxiv.org/oai?verb=GetRecord&identifier=oai:arXiv.org:2510.22581&metadataPrefix=arXiv",
      "asserted_by": "arXiv submitter",
      "captured_at": "2026-10-09T09:52:37+00:00",
      "source_system": "arxiv",
      "observed_value": "http://creativecommons.org/licenses/by-nc-sa/4.0/",
      "documentation_url": "https://info.arxiv.org/help/oa/index.html",
      "responsibility_url": "https://info.arxiv.org/help/license/index.html",
      "documentation_label": "arXiv OAI-PMH metadata documentation",
      "responsibility_note": "The submitter chooses the article-version licence and certifies the right to grant it."
    }
  },
  "descriptive": {
    "contributions": [
      {
        "name": "Kaushal Kumar Maurya",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Ekaterina Kochmar",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      }
    ],
    "identifiers": [
      {
        "scheme": "arxiv",
        "value": "2510.22581",
        "scope": "resource"
      }
    ],
    "dates": [
      {
        "meaning": "issued",
        "value": "2025-10-26",
        "scope": "resource"
      }
    ],
    "subjects": [
      {
        "value": "cs.CL",
        "scheme": "arxiv",
        "scope": "resource"
      }
    ],
    "relations": [],
    "educational": {
      "learning_resource_type": "",
      "intended_end_user_role": "",
      "context": "",
      "difficulty": "",
      "typical_learning_time": "",
      "description": ""
    },
    "container": {
      "kind": "",
      "uri": "",
      "title": "",
      "description": "",
      "identifier": ""
    },
    "declared_language": "en",
    "declared_type": "preprint",
    "rights_holder": "",
    "access_rights": "open",
    "publisher": "",
    "version": "v1",
    "citation": "",
    "notes": "",
    "lifecycle_status": "",
    "structure_role": "",
    "snapshot": {
      "resource": {
        "title": "Pedagogy-driven Evaluation of Generative AI-powered Intelligent Tutoring Systems",
        "authors": [
          "Kaushal Kumar Maurya",
          "Ekaterina Kochmar"
        ],
        "summary": "The interdisciplinary research domain of Artificial Intelligence in Education (AIED) has a long history of developing Intelligent Tutoring Systems (ITSs) by integrating insights from technological advancements, educational theories, and cognitive psychology. The remarkable success of generative AI (GenAI) models has accelerated the development of large language model (LLM)-powered ITSs, which have potential to imitate human-like, pedagogically rich, and cognitively demanding tutoring. However, the progress and impact of these systems remain largely untraceable due to the absence of reliable, universally accepted, and pedagogy-driven evaluation frameworks and benchmarks. Most existing educational dialogue-based ITS evaluations rely on subjective protocols and non-standardized benchmarks, leading to inconsistencies and limited generalizability. In this work, we take a step back from mainstream ITS development and provide comprehensive state-of-the-art evaluation practices, highlighting associated challenges through real-world case studies from careful and caring AIED research. Finally, building on insights from previous interdisciplinary AIED research, we propose three practical, feasible, and theoretically grounded research directions, rooted in learning science principles and aimed at establishing fair, unified, and scalable evaluation methodologies for ITSs.",
        "updated": "2025-10-26T08:44:21Z",
        "version": 1,
        "arxiv_id": "2510.22581",
        "published": "2025-10-26T08:44:21Z",
        "categories": [
          "cs.CL"
        ],
        "affiliations": [
          "",
          ""
        ],
        "primary_category": "cs.CL"
      }
    }
  },
  "representations": [
    {
      "role": "extracted",
      "media_type": "text/markdown",
      "size_bytes": 44988,
      "filename": "",
      "location": "",
      "source_format": "latexml-html",
      "converter": "arxiv-html"
    }
  ],
  "observed": {
    "language": {
      "method": "declared",
      "reason": "declared English",
      "accepted": true,
      "declared": "en",
      "detected": "en",
      "confidence": null,
      "review_required": false
    }
  },
  "collection": {
    "source": "arxiv",
    "source_label": "arXiv",
    "collected_at": "2026-10-09T09:52:40.406612+00:00",
    "query": "(all:\"artificial intelligence\" OR all:\"machine learning\" OR all:\"generative AI\" OR all:\"deep learning\" OR all:\"reinforcement learning\" OR all:\"large language model\") AND (all:\"AI concepts\" OR all:\"types of AI\" OR all:\"AI fundamentals\" OR all:\"recognizing AI\" OR all:\"recognising AI\" OR all:\"general versus narrow AI\" OR all:\"narrow AI\" OR all:\"general AI\" OR all:\"machine intelligence\" OR all:\"AI strengths and weaknesses\" OR all:\"traditional software\" OR all:\"rule-based systems\" OR all:\"introduction to AI\" OR all:\"introduction to artificial intelligence\" OR all:\"artificial intelligence introduction\" OR all:\"AI primer\" OR all:\"foundations of artificial intelligence\" OR all:\"overview of AI\" OR all:\"understanding AI\" OR all:\"history of AI\" OR all:\"AI essentials\" OR all:\"AI terminology\" OR all:\"metaphors for AI\" OR all:\"AI fundamental concepts\" OR all:\"AI key concepts\" OR all:\"philosophy of AI\" OR all:\"critical AI literacy\")",
    "note": "where this was collected from. Not a claim that the provider published or authored it"
  },
  "provenance": [
    {
      "field": "resource_id",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/resource_id"
    },
    {
      "field": "query",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/query"
    },
    {
      "field": "collected_from",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/source"
    },
    {
      "field": "title",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/title"
    },
    {
      "field": "description",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/summary"
    },
    {
      "field": "version",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "subjects",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/categories/0"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/0"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/1"
    },
    {
      "field": "dates",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "identifiers",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/arxiv_id"
    },
    {
      "field": "content_language",
      "scope": "resource",
      "method": "gate_observation",
      "evidence_ref": "gate#/language/declared"
    },
    {
      "field": "declared_language",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "licence",
      "scope": "resource",
      "method": "gate_observation",
      "evidence_ref": "gate#/licence/evidence"
    },
    {
      "field": "access_rights",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "representations",
      "scope": "resource",
      "method": "conversion",
      "evidence_ref": "conversion#/latexml-html"
    },
    {
      "field": "declared_type",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "educational",
      "scope": "resource",
      "method": "declared_type_mapping",
      "evidence_ref": "profile#/learning_resource_type/preprint"
    }
  ],
  "incidents": [
    {
      "code": "absent",
      "field": "rights_holder",
      "detail": "no rights holder is named at source; the licence is recorded without one rather than attributed to the platform that served it"
    },
    {
      "code": "absent",
      "field": "publisher",
      "detail": "the source named no publisher of the work; where it was collected from is recorded as collection provenance instead, which is a different claim"
    },
    {
      "code": "absent",
      "field": "educational",
      "detail": "the source declared no educational metadata — no resource type, audience, context, difficulty or learning time. Nothing here estimates them"
    }
  ],
  "inferred": [
    {
      "rule": "purpose.declared_type",
      "field": "purpose",
      "value": "reference",
      "method": "inferred",
      "evidence": {
        "declared_type": "preprint"
      },
      "rule_version": "purpose/1"
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2211.11772",
      "label": "arXiv:2211.11772",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 136,
            "basis": "reference",
            "literal": "arXiv:2211.11772"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2106.03873",
      "label": "arXiv:2106.03873",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 137,
            "basis": "reference",
            "literal": "arXiv:2106.03873"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2009.06978",
      "label": "arXiv:2009.06978",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 139,
            "basis": "reference",
            "literal": "arXiv:2009.06978"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2411.15594",
      "label": "arXiv:2411.15594",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 143,
            "basis": "reference",
            "literal": "arXiv:2411.15594"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2407.12687",
      "label": "arXiv:2407.12687",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 148,
            "basis": "reference",
            "literal": "arXiv:2407.12687"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/1609.02907",
      "label": "arXiv:1609.02907",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 150,
            "basis": "reference",
            "literal": "arXiv:1609.02907"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2305.14536",
      "label": "arXiv:2305.14536",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 154,
            "basis": "reference",
            "literal": "arXiv:2305.14536"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2502.18940",
      "label": "arXiv:2502.18940",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 155,
            "basis": "reference",
            "literal": "arXiv:2502.18940"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2406.13903",
      "label": "arXiv:2406.13903",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 168,
            "basis": "reference",
            "literal": "arXiv:2406.13903"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2204.09652",
      "label": "arXiv:2204.09652",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 175,
            "basis": "reference",
            "literal": "arXiv:2204.09652"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2205.07540",
      "label": "arXiv:2205.07540",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 177,
            "basis": "reference",
            "literal": "arXiv:2205.07540"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2412.16429",
      "label": "arXiv:2412.16429",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 178,
            "basis": "reference",
            "literal": "arXiv:2412.16429"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2410.03017",
      "label": "arXiv:2410.03017",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 184,
            "basis": "reference",
            "literal": "arXiv:2410.03017"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2402.02420",
      "label": "arXiv:2402.02420",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 185,
            "basis": "reference",
            "literal": "arXiv:2402.02420"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/1904.09675",
      "label": "arXiv:1904.09675",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 191,
            "basis": "reference",
            "literal": "arXiv:1904.09675"
          }
        ],
        "kind": "citation"
      }
    }
  ]
}