{
  "identifier": "tag:aim-pro.eu,2026:oer/9b5685ec0cd9/record",
  "profile": "aimpro-oer-profile/1",
  "normaliser": "oerlib.metadata/2",
  "status": "ready",
  "normalised_at": "2026-10-09T09:47:22+00:00",
  "resource": {
    "id": "9b5685ec0cd9",
    "identifier": "tag:aim-pro.eu,2026:oer/9b5685ec0cd9",
    "uri": "https://arxiv.org/abs/2412.14215",
    "title": "Generative AI Toolkit -- a framework for increasing the quality of LLM-based applications over their whole life cycle",
    "description": "As LLM-based applications reach millions of customers, ensuring their scalability and continuous quality improvement is critical for success. However, the current workflows for developing, maintaining, and operating (DevOps) these applications are predominantly manual, slow, and based on trial-and-error. With this paper we introduce the Generative AI Toolkit, which automates essential workflows over the whole life cycle of LLM-based applications. The toolkit helps to configure, test, continuously monitor and optimize Generative AI applications such as agents, thus significantly improving quality while shortening release cycles. We showcase the effectiveness of our toolkit on representative use cases, share best practices, and outline future enhancements. Since we are convinced that our Generative AI Toolkit is helpful for other teams, we are open sourcing it on and hope that others will use, forward, adapt and improve",
    "creators": [
      "Jens Kohl",
      "Luisa Gloger",
      "Rui Costa",
      "Otto Kruse",
      "Manuel P. Luitz",
      "David Katz",
      "Gonzalo Barbeito",
      "Markus Schweier",
      "Ryan French",
      "Jonas Schroeder",
      "Thomas Riedl",
      "Raphael Perri",
      "Youssef Mostafa"
    ],
    "keywords": [
      "cs.SE",
      "cs.AI"
    ],
    "kind": "html",
    "content_language": "en",
    "declared_language": "en",
    "word_count": 9095,
    "content_hash": "d91eab1a14bfc5af",
    "metadata_only": false
  },
  "licence": {
    "raw": "CC-BY-4.0",
    "spdx": "CC-BY-4.0",
    "uri": "http://creativecommons.org/licenses/by/4.0/",
    "verdict": "OPEN",
    "note": "open, permissive",
    "ingestable": true,
    "commercial_ok": true,
    "derivable": true,
    "share_alike": false,
    "evidence": {
      "scope": "this submitted version of the paper",
      "method": "arXiv OAI-PMH GetRecord (metadataPrefix=arXiv)",
      "schema": 1,
      "status": "captured",
      "location": "arXiv:arXiv/arXiv:license",
      "source_url": "https://oaipmh.arxiv.org/oai?verb=GetRecord&identifier=oai:arXiv.org:2412.14215&metadataPrefix=arXiv",
      "asserted_by": "arXiv submitter",
      "captured_at": "2026-10-09T09:47:12+00:00",
      "source_system": "arxiv",
      "observed_value": "http://creativecommons.org/licenses/by/4.0/",
      "documentation_url": "https://info.arxiv.org/help/oa/index.html",
      "responsibility_url": "https://info.arxiv.org/help/license/index.html",
      "documentation_label": "arXiv OAI-PMH metadata documentation",
      "responsibility_note": "The submitter chooses the article-version licence and certifies the right to grant it."
    }
  },
  "descriptive": {
    "contributions": [
      {
        "name": "Jens Kohl",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Luisa Gloger",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Rui Costa",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Otto Kruse",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Manuel P. Luitz",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "David Katz",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Gonzalo Barbeito",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Markus Schweier",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Ryan French",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Jonas Schroeder",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Thomas Riedl",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Raphael Perri",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      },
      {
        "name": "Youssef Mostafa",
        "role": "author",
        "affiliation": "",
        "identifier": "",
        "scope": "resource"
      }
    ],
    "identifiers": [
      {
        "scheme": "arxiv",
        "value": "2412.14215",
        "scope": "resource"
      }
    ],
    "dates": [
      {
        "meaning": "issued",
        "value": "2024-12-18",
        "scope": "resource"
      }
    ],
    "subjects": [
      {
        "value": "cs.SE",
        "scheme": "arxiv",
        "scope": "resource"
      },
      {
        "value": "cs.AI",
        "scheme": "arxiv",
        "scope": "resource"
      }
    ],
    "relations": [],
    "educational": {
      "learning_resource_type": "",
      "intended_end_user_role": "",
      "context": "",
      "difficulty": "",
      "typical_learning_time": "",
      "description": ""
    },
    "container": {
      "kind": "",
      "uri": "",
      "title": "",
      "description": "",
      "identifier": ""
    },
    "declared_language": "en",
    "declared_type": "preprint",
    "rights_holder": "",
    "access_rights": "open",
    "publisher": "",
    "version": "v1",
    "citation": "",
    "notes": "",
    "lifecycle_status": "",
    "structure_role": "",
    "snapshot": {
      "resource": {
        "title": "Generative AI Toolkit -- a framework for increasing the quality of LLM-based applications over their whole life cycle",
        "authors": [
          "Jens Kohl",
          "Luisa Gloger",
          "Rui Costa",
          "Otto Kruse",
          "Manuel P. Luitz",
          "David Katz",
          "Gonzalo Barbeito",
          "Markus Schweier",
          "Ryan French",
          "Jonas Schroeder",
          "Thomas Riedl",
          "Raphael Perri",
          "Youssef Mostafa"
        ],
        "summary": "As LLM-based applications reach millions of customers, ensuring their scalability and continuous quality improvement is critical for success. However, the current workflows for developing, maintaining, and operating (DevOps) these applications are predominantly manual, slow, and based on trial-and-error. With this paper we introduce the Generative AI Toolkit, which automates essential workflows over the whole life cycle of LLM-based applications. The toolkit helps to configure, test, continuously monitor and optimize Generative AI applications such as agents, thus significantly improving quality while shortening release cycles. We showcase the effectiveness of our toolkit on representative use cases, share best practices, and outline future enhancements. Since we are convinced that our Generative AI Toolkit is helpful for other teams, we are open sourcing it on and hope that others will use, forward, adapt and improve",
        "updated": "2024-12-18T10:40:00Z",
        "version": 1,
        "arxiv_id": "2412.14215",
        "published": "2024-12-18T10:40:00Z",
        "categories": [
          "cs.SE",
          "cs.AI"
        ],
        "affiliations": [
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          "",
          ""
        ],
        "primary_category": "cs.SE"
      }
    }
  },
  "representations": [
    {
      "role": "extracted",
      "media_type": "text/markdown",
      "size_bytes": 68475,
      "filename": "",
      "location": "",
      "source_format": "latexml-html",
      "converter": "arxiv-html"
    }
  ],
  "observed": {
    "language": {
      "method": "declared",
      "reason": "declared English",
      "accepted": true,
      "declared": "en",
      "detected": "en",
      "confidence": null,
      "review_required": false
    }
  },
  "collection": {
    "source": "arxiv",
    "source_label": "arXiv",
    "collected_at": "2026-10-09T09:47:22.083231+00:00",
    "query": "(all:\"artificial intelligence\" OR all:\"machine learning\" OR all:\"generative AI\" OR all:\"deep learning\" OR all:\"reinforcement learning\" OR all:\"large language model\") AND (all:\"AI concepts\" OR all:\"types of AI\" OR all:\"AI fundamentals\" OR all:\"recognizing AI\" OR all:\"recognising AI\" OR all:\"general versus narrow AI\" OR all:\"narrow AI\" OR all:\"general AI\" OR all:\"machine intelligence\" OR all:\"AI strengths and weaknesses\" OR all:\"traditional software\" OR all:\"rule-based systems\" OR all:\"introduction to AI\" OR all:\"introduction to artificial intelligence\" OR all:\"artificial intelligence introduction\" OR all:\"AI primer\" OR all:\"foundations of artificial intelligence\" OR all:\"overview of AI\" OR all:\"understanding AI\" OR all:\"history of AI\" OR all:\"AI essentials\" OR all:\"AI terminology\" OR all:\"metaphors for AI\" OR all:\"AI fundamental concepts\" OR all:\"AI key concepts\" OR all:\"philosophy of AI\" OR all:\"critical AI literacy\")",
    "note": "where this was collected from. Not a claim that the provider published or authored it"
  },
  "provenance": [
    {
      "field": "resource_id",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/resource_id"
    },
    {
      "field": "query",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/query"
    },
    {
      "field": "collected_from",
      "scope": "resource",
      "method": "engine",
      "evidence_ref": "engine#/source"
    },
    {
      "field": "title",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/title"
    },
    {
      "field": "description",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/summary"
    },
    {
      "field": "version",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "subjects",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/categories/0"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/0"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/1"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/2"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/3"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/4"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/5"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/6"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/7"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/8"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/9"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/10"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/11"
    },
    {
      "field": "contributions",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/authors/12"
    },
    {
      "field": "dates",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "identifiers",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource/arxiv_id"
    },
    {
      "field": "content_language",
      "scope": "resource",
      "method": "gate_observation",
      "evidence_ref": "gate#/language/declared"
    },
    {
      "field": "declared_language",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "licence",
      "scope": "resource",
      "method": "gate_observation",
      "evidence_ref": "gate#/licence/evidence"
    },
    {
      "field": "access_rights",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "representations",
      "scope": "resource",
      "method": "conversion",
      "evidence_ref": "conversion#/latexml-html"
    },
    {
      "field": "declared_type",
      "scope": "resource",
      "method": "source_metadata",
      "evidence_ref": "snapshot#/resource"
    },
    {
      "field": "educational",
      "scope": "resource",
      "method": "declared_type_mapping",
      "evidence_ref": "profile#/learning_resource_type/preprint"
    }
  ],
  "incidents": [
    {
      "code": "absent",
      "field": "rights_holder",
      "detail": "no rights holder is named at source; the licence is recorded without one rather than attributed to the platform that served it"
    },
    {
      "code": "absent",
      "field": "publisher",
      "detail": "the source named no publisher of the work; where it was collected from is recorded as collection provenance instead, which is a different claim"
    },
    {
      "code": "absent",
      "field": "educational",
      "detail": "the source declared no educational metadata — no resource type, audience, context, difficulty or learning time. Nothing here estimates them"
    }
  ],
  "inferred": [
    {
      "rule": "purpose.declared_type",
      "field": "purpose",
      "value": "reference",
      "method": "inferred",
      "evidence": {
        "declared_type": "preprint"
      },
      "rule_version": "purpose/1"
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2302.13971",
      "label": "arXiv:2302.13971",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 215,
            "basis": "reference",
            "literal": "arXiv:2302.13971"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2307.09288",
      "label": "arXiv:2307.09288",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 216,
            "basis": "reference",
            "literal": "arXiv:2307.09288"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2203.15556",
      "label": "arXiv:2203.15556",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 217,
            "basis": "reference",
            "literal": "arXiv:2203.15556"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2306.11644",
      "label": "arXiv:2306.11644",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 218,
            "basis": "reference",
            "literal": "arXiv:2306.11644"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2306.01116",
      "label": "arXiv:2306.01116",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 219,
            "basis": "reference",
            "literal": "arXiv:2306.01116"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2309.07864",
      "label": "arXiv:2309.07864",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 221,
            "basis": "reference",
            "literal": "arXiv:2309.07864"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2307.06435",
      "label": "arXiv:2307.06435",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 224,
            "basis": "reference",
            "literal": "arXiv:2307.06435"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2302.01560",
      "label": "arXiv:2302.01560",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 225,
            "basis": "reference",
            "literal": "arXiv:2302.01560"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2308.08155",
      "label": "arXiv:2308.08155",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 227,
            "basis": "reference",
            "literal": "arXiv:2308.08155"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/1804.07461",
      "label": "arXiv:1804.07461",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 230,
            "basis": "reference",
            "literal": "arXiv:1804.07461"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2308.04624",
      "label": "arXiv:2308.04624",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 231,
            "basis": "reference",
            "literal": "arXiv:2308.04624"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2206.15030",
      "label": "arXiv:2206.15030",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 233,
            "basis": "reference",
            "literal": "arXiv:2206.15030"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2104.07567",
      "label": "arXiv:2104.07567",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 235,
            "basis": "reference",
            "literal": "arXiv:2104.07567"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2307.15043",
      "label": "arXiv:2307.15043",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 238,
            "basis": "reference",
            "literal": "arXiv:2307.15043"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2108.07258",
      "label": "arXiv:2108.07258",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 247,
            "basis": "reference",
            "literal": "arXiv:2108.07258"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2310.03714",
      "label": "arXiv:2310.03714",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 249,
            "basis": "reference",
            "literal": "arXiv:2310.03714"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2411.03350",
      "label": "arXiv:2411.03350",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 254,
            "basis": "reference",
            "literal": "arXiv:2411.03350"
          }
        ],
        "kind": "citation"
      }
    },
    {
      "field": "references",
      "value": "https://arxiv.org/abs/2407.21075",
      "label": "arXiv:2407.21075",
      "resource_id": "",
      "method": "inferred",
      "rule": "citation",
      "rule_version": "links/1",
      "evidence": {
        "links": [
          {
            "how": "citation",
            "line": 255,
            "basis": "reference",
            "literal": "arXiv:2407.21075"
          }
        ],
        "kind": "citation"
      }
    }
  ]
}