{
  "abstract": "Methods, systems, and apparatus, including computer programs encoded on a computer storage medium, for controlling an agent interacting with an environment. In one aspect, a method comprises: receiving one or more observations of an environment; receiving an input text sequence that describes a task to be performed by a robot in the environment; generating an encoded representation of the input text sequence in an embedding space; generating a corresponding encoded representation of each of the one or more observations in the embedding space; generating a sequence of input tokens that comprises the encoded representation of the input text sequence and the corresponding encoded representation of each observation; processing the sequence of input tokens using a language model neural network to generate an output text sequence that comprises high-level natural language instructions; and determining, from the high-level natural language instructions, one or more actions to be performed by the robot.",
  "assignees": [
    "GOOGLE LLC"
  ],
  "authors": [
    "DRIESS DANNY MICHAEL",
    "FLORENCE PETER RAYMOND",
    "GREFF KLAUS",
    "MORDATCH IGOR",
    "SAJJADI SEYED MOHAMMAD MEHDI",
    "ZENG ANDY"
  ],
  "canonical_url": "https://opentechnical.org/p/OT-2026-000001865",
  "categories": [
    "robotics-automation/robot-architectures"
  ],
  "extent": {
    "figures": 0,
    "paragraphs": 0,
    "words": 149
  },
  "figures": [],
  "files": [
    {
      "bytes": 10134,
      "media_type": "text/html; charset=utf-8",
      "name": "index.html",
      "sha256": "0737f2635d7f190618311225c14481770f66054da002911dbfccf73229aa1c29"
    },
    {
      "bytes": 1459,
      "media_type": "text/markdown; charset=utf-8",
      "name": "publication.md",
      "sha256": "4eea926a4163e4d86cc7119d9b802ca3f340caf6c6c4e2d85f2928ddcd22eee9"
    },
    {
      "bytes": 1554,
      "media_type": "text/plain; charset=utf-8",
      "name": "publication.txt",
      "sha256": "f744581fa844757eabd0e8876dd8350cc9bc81194e4052010434c16a459cbe54"
    }
  ],
  "integrity": {
    "hash_algorithm": "sha256",
    "keys_url": "https://opentechnical.org/integrity/keys",
    "record_url": "https://opentechnical.org/p/OT-2026-000001865/integrity",
    "signature_algorithm": "Ed25519",
    "signature_file": "manifest.sig",
    "verify_url": "https://opentechnical.org/integrity/verify"
  },
  "keywords": [
    "b25j9/1658",
    "b25j9/1658",
    "g06n3/045",
    "g06n3/045",
    "b25j9/16",
    "b25j9/161"
  ],
  "language": "en",
  "license": {
    "name": "Creative Commons Attribution 4.0 International",
    "url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "publication_id": "OT-2026-000001865",
  "published_at": "2026-09-15T19:09:09.091488Z",
  "publisher": "Nebula Innovations Limited",
  "record_type": "record",
  "schema": "opentechnical-publication-manifest/1.0",
  "source_date": "2026-03-17",
  "source_id": "US-12576523-B2",
  "source_name": "the United States Patent and Trademark Office",
  "source_url": "https://patents.google.com/patent/US12576523B2",
  "supersedes": null,
  "title": "Controlling robots using multi-modal language models",
  "uuid": "01a0a679-1863-7093-b617-d21c3ddc0b43"
}