[ { "section": "Section A - Evidence to look for before adoption", "source": "The published evaluation or the product documentation. This section should be answered by reading the paper or the documentation about the model.", "items": [ { "statement_id": "1.8", "agreement": "100%", "question": "Does the published evaluation report accuracy separately for each task the system will be used for?", "statement": "Clinical accuracy should be reported separately for each intended task (e.g. diagnosis, treatment recommendation, toxicity management, biomarker interpretation, clinical trial matching, molecular tumor board support), as performance may vary substantially across tasks." }, { "statement_id": "1.14", "agreement": "96.6%", "question": "Does it report performance on real clinical records, or only on standardised clinical vignettes?", "statement": "AI system performance should be evaluated on both standardized clinical vignettes and real-world, unstructured clinical data (e.g. actual electronic health records with missing or inconsistent information) to assess robustness under realistic conditions. The scope of validation (e.g. per institution, per EHR system, or per clinical setting) should be specified and justified." }, { "statement_id": "2.1", "agreement": "100%", "question": "Does it report a hallucination rate?", "statement": "Evaluation studies should report the hallucination rate, defined as outputs that contain fabricated clinical information not supported by the input data or established medical knowledge (e.g. citing a non-existent trial, inventing a drug dose, fabricating a molecular finding, or misinterpreting an image)." }, { "statement_id": "2.2", "agreement": "86.2%", "question": "Does it grade potentially harmful outputs on a defined harm classification scale, rather than describing them narratively?", "statement": "A standardized harm classification scale for clinical AI outputs should be adopted (e.g. no harm / minor harm / moderate harm / severe or potentially lethal harm), with definitions specific to oncology decision-making. Where applicable, established clinical scales (e.g. CTCAE) should inform the grading." }, { "statement_id": "3.1", "agreement": "92.6%", "question": "Does it report performance broken down by patient subgroup, at minimum age and sex, and declare any missing demographic data?", "statement": "AI evaluation studies should report performance stratified by clinically and demographically relevant patient variables, including at minimum age and sex. Where data are available, additional stratification should include race or ethnicity, socioeconomic factors, and geographic region. Missing demographic data should be reported transparently." }, { "statement_id": "4.1", "agreement": "100%", "question": "Does it identify exactly which system was tested — model name, version, date of access, and how it was prompted?", "statement": "Every published AI evaluation study should disclose the specific system(s) evaluated (including model name, version, and date of access), the prompting or input strategy employed, and any additional methods applied (e.g. fine-tuning on medical data, connecting the system to external knowledge sources, or integration of multiple data modalities)." }, { "statement_id": "4.4", "agreement": "96.6%", "question": "Does it report whether repeated identical queries produce consistent answers?", "statement": "Evaluation studies should report the reproducibility of AI outputs: whether repeated queries with identical clinical inputs produce consistent answers, or whether outputs vary meaningfully across runs." }, { "statement_id": "8.6", "agreement": "93.1%", "question": "Does it assess the risk that the system had already seen the evaluation cases during training?", "statement": "The risk of data contamination (i.e. the possibility that the AI system was trained on the same clinical cases used for evaluation) should be assessed and reported in all evaluation studies, particularly those using published clinical vignettes or board-style examination questions." }, { "statement_id": "5.1", "agreement": "79.3%", "question": "Does it include validation at an institution other than the one that developed the system?", "statement": "AI system performance should be validated in at least one external institution (external validation) before the tool is recommended for routine clinical use. The scope of external validation should account for relevant differences in clinical infrastructure, including the electronic health record system in use." }, { "statement_id": "5.2", "agreement": "82.8%", "question": "Does the evidence go beyond retrospective cases into a controlled prospective setting?", "statement": "AI evaluation should follow a three-stage validation pathway: (1) retrospective evaluation on historical clinical cases, (2) controlled prospective evaluation in a clinical setting, (3) monitored real-world deployment with ongoing outcome tracking." }, { "statement_id": "8.10", "agreement": "82.8%", "question": "Is the supporting evidence drawn from oncology, rather than from other specialties alone?", "statement": "Evidence from AI evaluation studies conducted outside oncology (e.g. in general medicine, cardiology, or other specialties) should not be considered sufficient for recommending deployment in oncology without dedicated oncology-specific validation, given the unique complexity and risk profile of cancer treatment decisions." } ] }, { "section": "Section B - Institutional requirements", "source": "Institutional policies. Each question should be read in regard to the specific system you are assessing", "items": [ { "statement_id": "7.6", "agreement": "100%", "question": "Does this system transmit patient data outside the institution, and are the categories of data it requires permitted under our institutional rules?", "statement": "Institutions should define which categories of patient information may be submitted to external commercial systems vs. only to locally deployed institutional systems." }, { "statement_id": "7.1", "agreement": "89.7%", "question": "Is this system covered by a documented institutional governance decision, rather than being used informally?", "statement": "Institutions should establish formal policies that either integrate validated AI tools into clinical workflows under defined governance, or clearly define acceptable boundaries for individual, unofficial use of external AI systems by clinicians." }, { "statement_id": "4.8", "agreement": "82.8%", "question": "Is it documented whether this system is locally hosted or cloud-based, and is that choice recorded as a governance decision with its reasoning?", "statement": "The choice between deploying an open-source (locally hosted) system vs. a cloud-based commercial system should be documented as an institutional governance decision, with explicit consideration of data privacy, regulatory compliance, and the level of institutional control over the system." }, { "statement_id": "7.5", "agreement": "86.2%", "question": "Is the regulatory status of this system in our jurisdiction documented ?", "statement": "The regulatory classification of clinical AI systems should be explicitly considered in the evaluation and deployment framework." }, { "statement_id": "7.4", "agreement": "89.7%", "question": "Does the documentation for this system state that it functions as decision support, with final responsibility for the clinical decision remaining with the treating clinician?", "statement": "Responsibility for clinical decisions should remain with the treating clinician, regardless of whether an AI system was used in the decision-making process." }, { "statement_id": "7.3", "agreement": "81.5%", "question": "Is there a documented arrangement for informing patients when this system contributes to decisions about their care?", "statement": "When AI systems are formally integrated into institutional clinical workflows, institutions should disclose to patients that AI-assisted decision support is being used as part of their care." }, { "statement_id": "2.11", "agreement": "89.7%", "question": "Is there a documented fallback procedure for when this system is unavailable or produces unreliable output?", "statement": "Institutions deploying clinical AI systems should ensure that adequate clinical expertise and fallback procedures are available in case the AI system becomes unavailable or produces unreliable outputs due to technical failure, system errors, or missed updates." } ] }, { "section": "Section C - Evidence to obtain from the vendor", "source": "The supplier's documentation, or contract. It refers to the information that the vendor gives about the model.", "items": [ { "statement_id": "5.9", "agreement": "96.6%", "question": "Does the vendor provide evidence of a post-deployment safety monitoring arrangement, including structured feedback collection from clinical users?", "statement": "Vendors of clinical AI products should be required to support structured feedback collection from clinical users and to participate in post-deployment safety monitoring, analogous to existing post-market surveillance requirements for medical devices." }, { "statement_id": "5.8", "agreement": "82.8%", "question": "Does the vendor provide a written division of responsibility between supplier and institution for continuous quality and safety monitoring?", "statement": "The responsibility for continuous quality monitoring and safety surveillance of clinical AI systems should be shared between the vendor or developer and the deploying institution, with clearly defined roles for each party." }, { "statement_id": "5.10", "agreement": "92.6%", "question": "Does the vendor provide a reporting channel through which a clinician can report an error or unexpected output to both the institution and the developer?", "statement": "A structured reporting channel should be established for clinicians to report errors, unexpected outputs, and safety concerns to both institutional quality assurance and the AI system developer, enabling continuous post-deployment monitoring and early detection of systematic safety signals." }, { "statement_id": "5.6", "agreement": "93.1%", "question": "Does the vendor provide evidence that the validation shown applies to the version we will actually be running?", "statement": "Validation of a specific AI model version should not automatically extend to successor versions or substantially updated releases. Each major version update should undergo independent evaluation proportionate to the scope of the changes." } ] }, { "section": "Section D - Once the system is in use", "source": "Local monitoring records and local practice. It refers to systems already deployed. It should be answered from what can be observed in your own department, and repeated at each re-evaluation.", "items": [ { "statement_id": "5.3", "agreement": "100%", "question": "Is there documented monitoring in place to detect performance falling off as guidelines change and new drugs are approved?", "statement": "After deployment, systematic monitoring should be in place to detect performance degradation over time, which may occur as medical knowledge evolves, new drugs are approved, or clinical guidelines are updated while the system's training data remains static." }, { "statement_id": "5.4", "agreement": "96.6%", "question": "Is there a documented date, or trigger, at which this system will be formally re-evaluated?", "statement": "A defined schedule for re-evaluation of deployed AI systems should be established, for example triggered by major guideline updates, new drug approvals, changes in standard-of-care, or at regular intervals (e.g. annually)." }, { "statement_id": "5.5", "agreement": "96.6%", "question": "Are there documented rules for what happens when the vendor releases a new version, and a named person managing that?", "statement": "Clear rules should be established for when and how a deployed AI system is updated to a new version, and what level of re-validation is required before the new version replaces the previous one in clinical use. Institutions should have dedicated expertise available to manage this process." }, { "statement_id": "5.7", "agreement": "89.7%", "question": "Is there a documented arrangement linking AI-supported decisions to what actually happened to those patients?", "statement": "Post-deployment monitoring should, where feasible, link AI-supported clinical decisions to actual patient outcomes (e.g. treatment response rates, progression-free survival, adverse event rates) over time, to assess the real-world impact of the tool." }, { "statement_id": "6.9", "agreement": "89.7%", "question": "Is there evidence that over-reliance (i.e. clinicians accept incorrect outputs without independent review) is being watched for?", "statement": "Evaluation studies should assess the risk of over-reliance (automatic bias) on AI, including clinician acceptance of incorrect outputs without adequate independent review." }, { "statement_id": "5.11", "agreement": "86.2%", "question": "Is there evidence that output remains stable across our hardware, software versions and interface updates?", "statement": "AI system outputs should demonstrate acceptable stability across different technical configurations (e.g. hardware, software versions, API updates) without clinically meaningful changes in recommendations." } ] } ]