{ "identifier": "tag:aim-pro.eu,2026:oer/8b4ef701d9f2/record", "profile": "aimpro-oer-profile/1", "normaliser": "oerlib.metadata/2", "status": "ready", "normalised_at": "2026-10-09T09:51:05+00:00", "resource": { "id": "8b4ef701d9f2", "identifier": "tag:aim-pro.eu,2026:oer/8b4ef701d9f2", "uri": "https://arxiv.org/abs/2306.05949", "title": "Evaluating the Social Impact of Generative AI Systems in Systems and Society", "description": "Generative AI systems across modalities, ranging from text (including code), image, audio, and video, have broad social impacts, but there is no official standard for means of evaluating those impacts or for which impacts should be evaluated. In this paper, we present a guide that moves toward a standard approach in evaluating a base generative AI system for any modality in two overarching categories: what can be evaluated in a base system independent of context and what can be evaluated in a societal context. Importantly, this refers to base systems that have no predetermined application or deployment context, including a model itself, as well as system components, such as training data. Our framework for a base system defines seven categories of social impact: bias, stereotypes, and representational harms; cultural values and sensitive content; disparate performance; privacy and data protection; financial costs; environmental costs; and data and content moderation labor costs. Suggested methods for evaluation apply to listed generative modalities and analyses of the limitations of existing evaluations serve as a starting point for necessary investment in future evaluations. We offer five overarching categories for what can be evaluated in a broader societal context, each with its own subcategories: trustworthiness and autonomy; inequality, marginalization, and violence; concentration of authority; labor and creativity; and ecosystem and environment. Each subcategory includes recommendations for mitigating harm.", "creators": [ "Irene Solaiman", "Zeerak Talat", "William Agnew", "Lama Ahmad", "Dylan Baker", "Su Lin Blodgett", "Canyu Chen", "Hal Daumé", "Jesse Dodge", "Isabella Duan", "Ellie Evans", "Felix Friedrich", "Avijit Ghosh", "Usman Gohar", "Sara Hooker", "Yacine Jernite", "Ria Kalluri", "Alberto Lusoli", "Alina Leidinger", "Michelle Lin", "Xiuzhu Lin", "Sasha Luccioni", "Jennifer Mickel", "Margaret Mitchell", "Jessica Newman", "Anaelia Ovalle", "Marie-Therese Png", "Shubham Singh", "Andrew Strait", "Lukas Struppek", "Arjun Subramonian" ], "keywords": [ "cs.CY", "cs.AI" ], "kind": "text", "content_language": "en", "declared_language": "en", "word_count": 342, "content_hash": "d665a44d84b1ae4d", "metadata_only": true }, "licence": { "raw": "CC-BY-SA-4.0", "spdx": "CC-BY-SA-4.0", "uri": "http://creativecommons.org/licenses/by-sa/4.0/", "verdict": "SHARE_ALIKE", "note": "open but ShareAlike/copyleft — derivatives keep the license", "ingestable": true, "commercial_ok": true, "derivable": true, "share_alike": true, "evidence": { "scope": "this submitted version of the paper", "method": "arXiv OAI-PMH GetRecord (metadataPrefix=arXiv)", "schema": 1, "status": "captured", "location": "arXiv:arXiv/arXiv:license", "source_url": "https://oaipmh.arxiv.org/oai?verb=GetRecord&identifier=oai:arXiv.org:2306.05949&metadataPrefix=arXiv", "asserted_by": "arXiv submitter", "captured_at": "2026-10-09T09:50:59+00:00", "source_system": "arxiv", "observed_value": "http://creativecommons.org/licenses/by-sa/4.0/", "documentation_url": "https://info.arxiv.org/help/oa/index.html", "responsibility_url": "https://info.arxiv.org/help/license/index.html", "documentation_label": "arXiv OAI-PMH metadata documentation", "responsibility_note": "The submitter chooses the article-version licence and certifies the right to grant it." } }, "descriptive": { "contributions": [ { "name": "Irene Solaiman", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Zeerak Talat", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "William Agnew", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Lama Ahmad", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Dylan Baker", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Su Lin Blodgett", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Canyu Chen", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Hal Daumé", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Jesse Dodge", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Isabella Duan", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Ellie Evans", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Felix Friedrich", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Avijit Ghosh", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Usman Gohar", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Sara Hooker", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Yacine Jernite", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Ria Kalluri", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Alberto Lusoli", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Alina Leidinger", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Michelle Lin", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Xiuzhu Lin", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Sasha Luccioni", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Jennifer Mickel", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Margaret Mitchell", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Jessica Newman", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Anaelia Ovalle", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Marie-Therese Png", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Shubham Singh", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Andrew Strait", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Lukas Struppek", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" }, { "name": "Arjun Subramonian", "role": "author", "affiliation": "", "identifier": "", "scope": "resource" } ], "identifiers": [ { "scheme": "arxiv", "value": "2306.05949", "scope": "resource" }, { "scheme": "doi", "value": "10.1093/oxfordhb/9780198940272.013.0025", "scope": "resource" } ], "dates": [ { "meaning": "issued", "value": "2023-06-09", "scope": "resource" }, { "meaning": "modified", "value": "2024-06-28", "scope": "resource" } ], "subjects": [ { "value": "cs.CY", "scheme": "arxiv", "scope": "resource" }, { "value": "cs.AI", "scheme": "arxiv", "scope": "resource" } ], "relations": [], "educational": { "learning_resource_type": "", "intended_end_user_role": "", "context": "", "difficulty": "", "typical_learning_time": "", "description": "" }, "container": { "kind": "", "uri": "", "title": "", "description": "", "identifier": "" }, "declared_language": "en", "declared_type": "preprint", "rights_holder": "", "access_rights": "open", "publisher": "", "version": "v4", "citation": "The Oxford Handbook of the Foundations and Regulation of Generative AI, 18 December 2025", "notes": "", "lifecycle_status": "revised", "structure_role": "", "snapshot": { "resource": { "doi": "10.1093/oxfordhb/9780198940272.013.0025", "title": "Evaluating the Social Impact of Generative AI Systems in Systems and Society", "authors": [ "Irene Solaiman", "Zeerak Talat", "William Agnew", "Lama Ahmad", "Dylan Baker", "Su Lin Blodgett", "Canyu Chen", "Hal Daumé", "Jesse Dodge", "Isabella Duan", "Ellie Evans", "Felix Friedrich", "Avijit Ghosh", "Usman Gohar", "Sara Hooker", "Yacine Jernite", "Ria Kalluri", "Alberto Lusoli", "Alina Leidinger", "Michelle Lin", "Xiuzhu Lin", "Sasha Luccioni", "Jennifer Mickel", "Margaret Mitchell", "Jessica Newman", "Anaelia Ovalle", "Marie-Therese Png", "Shubham Singh", "Andrew Strait", "Lukas Struppek", "Arjun Subramonian" ], "summary": "Generative AI systems across modalities, ranging from text (including code), image, audio, and video, have broad social impacts, but there is no official standard for means of evaluating those impacts or for which impacts should be evaluated. In this paper, we present a guide that moves toward a standard approach in evaluating a base generative AI system for any modality in two overarching categories: what can be evaluated in a base system independent of context and what can be evaluated in a societal context. Importantly, this refers to base systems that have no predetermined application or deployment context, including a model itself, as well as system components, such as training data. Our framework for a base system defines seven categories of social impact: bias, stereotypes, and representational harms; cultural values and sensitive content; disparate performance; privacy and data protection; financial costs; environmental costs; and data and content moderation labor costs. Suggested methods for evaluation apply to listed generative modalities and analyses of the limitations of existing evaluations serve as a starting point for necessary investment in future evaluations. We offer five overarching categories for what can be evaluated in a broader societal context, each with its own subcategories: trustworthiness and autonomy; inequality, marginalization, and violence; concentration of authority; labor and creativity; and ecosystem and environment. Each subcategory includes recommendations for mitigating harm.", "updated": "2024-06-28T13:50:57Z", "version": 4, "arxiv_id": "2306.05949", "published": "2023-06-09T15:05:13Z", "categories": [ "cs.CY", "cs.AI" ], "journal_ref": "The Oxford Handbook of the Foundations and Regulation of Generative AI, 18 December 2025", "affiliations": [ "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "primary_category": "cs.CY" } } }, "representations": [ { "role": "extracted", "media_type": "text/markdown", "size_bytes": 2373, "filename": "", "location": "", "source_format": "abstract", "converter": "abstract" } ], "observed": { "language": { "method": "declared", "reason": "declared English", "accepted": true, "declared": "en", "detected": "en", "confidence": null, "review_required": false } }, "collection": { "source": "arxiv", "source_label": "arXiv", "collected_at": "2026-10-09T09:51:05.942129+00:00", "query": "(all:\"artificial intelligence\" OR all:\"machine learning\" OR all:\"generative AI\" OR all:\"deep learning\" OR all:\"reinforcement learning\" OR all:\"large language model\") AND (all:\"AI concepts\" OR all:\"types of AI\" OR all:\"AI fundamentals\" OR all:\"recognizing AI\" OR all:\"recognising AI\" OR all:\"general versus narrow AI\" OR all:\"narrow AI\" OR all:\"general AI\" OR all:\"machine intelligence\" OR all:\"AI strengths and weaknesses\" OR all:\"traditional software\" OR all:\"rule-based systems\" OR all:\"introduction to AI\" OR all:\"introduction to artificial intelligence\" OR all:\"artificial intelligence introduction\" OR all:\"AI primer\" OR all:\"foundations of artificial intelligence\" OR all:\"overview of AI\" OR all:\"understanding AI\" OR all:\"history of AI\" OR all:\"AI essentials\" OR all:\"AI terminology\" OR all:\"metaphors for AI\" OR all:\"AI fundamental concepts\" OR all:\"AI key concepts\" OR all:\"philosophy of AI\" OR all:\"critical AI literacy\")", "note": "where this was collected from. Not a claim that the provider published or authored it" }, "provenance": [ { "field": "resource_id", "scope": "resource", "method": "engine", "evidence_ref": "engine#/resource_id" }, { "field": "query", "scope": "resource", "method": "engine", "evidence_ref": "engine#/query" }, { "field": "collected_from", "scope": "resource", "method": "engine", "evidence_ref": "engine#/source" }, { "field": "title", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/title" }, { "field": "description", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/summary" }, { "field": "citation", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/journal_ref" }, { "field": "version", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "lifecycle_status", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "subjects", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/categories/0" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/0" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/1" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/2" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/3" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/4" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/5" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/6" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/7" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/8" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/9" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/10" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/11" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/12" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/13" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/14" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/15" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/16" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/17" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/18" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/19" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/20" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/21" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/22" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/23" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/24" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/25" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/26" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/27" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/28" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/29" }, { "field": "contributions", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/authors/30" }, { "field": "dates", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "identifiers", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/arxiv_id" }, { "field": "identifiers", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource/doi" }, { "field": "content_language", "scope": "resource", "method": "gate_observation", "evidence_ref": "gate#/language/declared" }, { "field": "declared_language", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "licence", "scope": "resource", "method": "gate_observation", "evidence_ref": "gate#/licence/evidence" }, { "field": "access_rights", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "representations", "scope": "resource", "method": "conversion", "evidence_ref": "conversion#/abstract" }, { "field": "declared_type", "scope": "resource", "method": "source_metadata", "evidence_ref": "snapshot#/resource" }, { "field": "educational", "scope": "resource", "method": "declared_type_mapping", "evidence_ref": "profile#/learning_resource_type/preprint" } ], "incidents": [ { "code": "absent", "field": "rights_holder", "detail": "no rights holder is named at source; the licence is recorded without one rather than attributed to the platform that served it" }, { "code": "absent", "field": "publisher", "detail": "the source named no publisher of the work; where it was collected from is recorded as collection provenance instead, which is a different claim" }, { "code": "absent", "field": "educational", "detail": "the source declared no educational metadata — no resource type, audience, context, difficulty or learning time. Nothing here estimates them" } ], "inferred": [ { "field": "referenced_by", "value": "https://arxiv.org/abs/2410.23704", "label": "Using Scenario-Writing for Identifying and Mitigating Impacts of Generative AI", "resource_id": "e5085a102e9f", "method": "inferred", "rule": "citation", "rule_version": "links/1", "evidence": { "links": [ { "how": "citation", "line": 76, "basis": "reference", "literal": "arXiv:2306.05949" } ], "kind": "citation" } } ] }