{
  "lom": {
    "general": {
      "identifier": [
        {
          "catalog": "URI",
          "entry": "tag:aim-pro.eu,2026:oer/e71a9e5763e5"
        },
        {
          "catalog": "URI",
          "entry": "https://arxiv.org/abs/2510.22581"
        },
        {
          "catalog": "ARXIV",
          "entry": "2510.22581"
        }
      ],
      "title": [
        {
          "language": "en",
          "string": "Pedagogy-driven Evaluation of Generative AI-powered Intelligent Tutoring Systems"
        }
      ],
      "language": [
        "en"
      ],
      "description": [
        {
          "language": "en",
          "string": "The interdisciplinary research domain of Artificial Intelligence in Education (AIED) has a long history of developing Intelligent Tutoring Systems (ITSs) by integrating insights from technological advancements, educational theories, and cognitive psychology. The remarkable success of generative AI (GenAI) models has accelerated the development of large language model (LLM)-powered ITSs, which have potential to imitate human-like, pedagogically rich, and cognitively demanding tutoring. However, the progress and impact of these systems remain largely untraceable due to the absence of reliable, universally accepted, and pedagogy-driven evaluation frameworks and benchmarks. Most existing educational dialogue-based ITS evaluations rely on subjective protocols and non-standardized benchmarks, leading to inconsistencies and limited generalizability. In this work, we take a step back from mainstream ITS development and provide comprehensive state-of-the-art evaluation practices, highlighting associated challenges through real-world case studies from careful and caring AIED research. Finally, building on insights from previous interdisciplinary AIED research, we propose three practical, feasible, and theoretically grounded research directions, rooted in learning science principles and aimed at establishing fair, unified, and scalable evaluation methodologies for ITSs."
        }
      ],
      "keyword": [
        {
          "language": "en",
          "string": "cs.CL"
        }
      ]
    },
    "lifeCycle": {
      "version": [
        {
          "language": "en",
          "string": "v1"
        }
      ],
      "contribute": [
        {
          "role": {
            "source": "LOMv1.0",
            "value": "author"
          },
          "entity": [
            "Kaushal Kumar Maurya"
          ],
          "date": {
            "dateTime": "2025-10-26"
          }
        },
        {
          "role": {
            "source": "LOMv1.0",
            "value": "author"
          },
          "entity": [
            "Ekaterina Kochmar"
          ],
          "date": {
            "dateTime": "2025-10-26"
          }
        }
      ]
    },
    "metaMetadata": {
      "identifier": [
        {
          "catalog": "URI",
          "entry": "tag:aim-pro.eu,2026:oer/e71a9e5763e5/record"
        }
      ],
      "contribute": [
        {
          "role": {
            "source": "LOMv1.0",
            "value": "creator"
          },
          "entity": [
            "AIM-PRO WP3 OER harvester (arXiv)"
          ],
          "date": {
            "dateTime": "2026-10-09T09:52:40.406612+00:00",
            "description": [
              {
                "language": "en",
                "string": "Date this metadata record was generated from the source response, not a date about the resource"
              }
            ]
          }
        }
      ],
      "metadataSchema": [
        "LOMv1.0",
        "aimpro-oer-profile/1"
      ],
      "language": "en"
    },
    "technical": {
      "format": [
        "text/markdown"
      ],
      "size": "44988",
      "location": [
        "https://arxiv.org/abs/2510.22581"
      ]
    },
    "educational": {
      "learningResourceType": [
        {
          "source": "LOMv1.0",
          "value": "narrative text"
        }
      ]
    },
    "rights": {
      "cost": {
        "source": "LOMv1.0",
        "value": "no"
      },
      "copyrightAndOtherRestrictions": {
        "source": "LOMv1.0",
        "value": "yes"
      },
      "description": [
        {
          "language": "en",
          "string": "CC-BY-NC-SA-4.0 · http://creativecommons.org/licenses/by-nc-sa/4.0/. Conditions: adaptation permitted, non-commercial use only, adaptations must carry the same licence, attribution required."
        }
      ]
    }
  },
  "inferred": {
    "relation": [
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2211.11772"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2211.11772"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 136,
              "basis": "reference",
              "literal": "arXiv:2211.11772"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2106.03873"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2106.03873"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 137,
              "basis": "reference",
              "literal": "arXiv:2106.03873"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2009.06978"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2009.06978"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 139,
              "basis": "reference",
              "literal": "arXiv:2009.06978"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2411.15594"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2411.15594"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 143,
              "basis": "reference",
              "literal": "arXiv:2411.15594"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2407.12687"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2407.12687"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 148,
              "basis": "reference",
              "literal": "arXiv:2407.12687"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/1609.02907"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:1609.02907"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 150,
              "basis": "reference",
              "literal": "arXiv:1609.02907"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2305.14536"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2305.14536"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 154,
              "basis": "reference",
              "literal": "arXiv:2305.14536"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2502.18940"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2502.18940"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 155,
              "basis": "reference",
              "literal": "arXiv:2502.18940"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2406.13903"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2406.13903"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 168,
              "basis": "reference",
              "literal": "arXiv:2406.13903"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2204.09652"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2204.09652"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 175,
              "basis": "reference",
              "literal": "arXiv:2204.09652"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2205.07540"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2205.07540"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 177,
              "basis": "reference",
              "literal": "arXiv:2205.07540"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2412.16429"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2412.16429"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 178,
              "basis": "reference",
              "literal": "arXiv:2412.16429"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2410.03017"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2410.03017"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 184,
              "basis": "reference",
              "literal": "arXiv:2410.03017"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/2402.02420"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:2402.02420"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 185,
              "basis": "reference",
              "literal": "arXiv:2402.02420"
            }
          ],
          "kind": "citation"
        }
      },
      {
        "kind": {
          "source": "LOMv1.0",
          "value": "references"
        },
        "resource": {
          "identifier": [
            {
              "catalog": "URI",
              "entry": "https://arxiv.org/abs/1904.09675"
            }
          ],
          "description": [
            {
              "language": "en",
              "string": "arXiv:1904.09675"
            }
          ]
        },
        "method": "inferred",
        "rule": "citation",
        "rule_version": "links/1",
        "evidence": {
          "links": [
            {
              "how": "citation",
              "line": 191,
              "basis": "reference",
              "literal": "arXiv:1904.09675"
            }
          ],
          "kind": "citation"
        }
      }
    ]
  }
}