{"id":1123773,"date":"2025-02-25T11:33:38","date_gmt":"2025-02-25T19:33:38","guid":{"rendered":"https:\/\/www.microsoft.com\/en-us\/research\/?post_type=msr-blog-post&#038;p=1123773"},"modified":"2025-02-25T15:06:25","modified_gmt":"2025-02-25T23:06:25","slug":"belief-state-transformers","status":"publish","type":"msr-blog-post","link":"https:\/\/www.microsoft.com\/en-us\/research\/articles\/belief-state-transformers\/","title":{"rendered":"Belief state transformers"},"content":{"rendered":"\n<p class=\"has-purple-color has-text-color has-link-color wp-elements-08be367eea732a066e06b254579d1ba0\"><em>Presented by&nbsp;<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/jcl\/\">John Langford<\/a>&nbsp;at&nbsp;<strong>Microsoft Research Forum, February 2025<\/strong><\/em><\/p>\n\n\n\n<div class=\"wp-block-media-text has-vertical-margin-none  has-vertical-padding-none  is-stacked-on-mobile has-white-background-color has-background\" style=\"grid-template-columns:25% auto\" data-bi-an=\"media-text\"><figure class=\"wp-block-media-text__media\"><img loading=\"lazy\" decoding=\"async\" width=\"360\" height=\"360\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360.jpg\" alt=\"Portrait of John Langford\" class=\"wp-image-705793 size-full\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360.jpg 360w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360-300x300.jpg 300w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360-150x150.jpg 150w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360-12x12.jpg 12w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2020\/10\/John-Langford_360x360-180x180.jpg 180w\" sizes=\"auto, (max-width: 360px) 100vw, 360px\" \/><\/figure><div class=\"wp-block-media-text__content\" data-bi-an=\"media-text\">\n<blockquote class=\"wp-block-quote is-style-spectrum is-layout-flow wp-block-quote-is-layout-flow\">\n<p>\u201cThat ability to condition on generation, rather than evaluate the generation, ends up being amazingly useful in terms of giving you a more honest valuation of the generated text.&#8221;<\/p>\n<cite><em>\u2013<\/em> John Langford, Partner Research Manager, Microsoft Research AI Frontiers<\/cite><\/blockquote>\n<\/div><\/div>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"Belief state transformers | Microsoft Research Forum\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube-nocookie.com\/embed\/aqhbRtB2Fyg?feature=oembed&rel=0\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<div class=\"annotations \" data-bi-aN=\"margin-callout\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 annotations__list--right\">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=What%20advantages%20do%20belief%20state%20transformers%20offer%20over%20traditional%20GPT-style%20models%20in%20planning%20and%20decision-making%20tasks?\" target=\"_blank\" aria-label=\"What advantages do belief state transformers offer over traditional GPT-style models in planning and decision-making tasks?\" data-bi-type=\"annotated-link\" data-bi-cN=\"What advantages do belief state transformers offer over traditional GPT-style models in planning and decision-making tasks?\" class=\"annotations__list-thumbnail\" >\n\t\t\t\t\t<img loading=\"lazy\" decoding=\"async\" width=\"172\" height=\"96\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-240x135.png\" class=\"mb-2\" alt=\"Ask Microsoft research copilot experience\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-240x135.png 240w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-300x169.png 300w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-1024x576.png 1024w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-768x432.png 768w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-1066x600.png 1066w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-655x368.png 655w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-343x193.png 343w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-640x360.png 640w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-960x540.png 960w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo-1280x720.png 1280w, https:\/\/www.microsoft.com\/en-us\/research\/wp-content\/uploads\/2024\/01\/MSR-Chat-Promo.png 1400w\" sizes=\"auto, (max-width: 172px) 100vw, 172px\" \/>\t\t\t\t<\/a>\n\t\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Microsoft research copilot experience<\/span>\n\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=What%20advantages%20do%20belief%20state%20transformers%20offer%20over%20traditional%20GPT-style%20models%20in%20planning%20and%20decision-making%20tasks?\" target=\"_blank\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"What advantages do belief state transformers offer over traditional GPT-style models in planning and decision-making tasks?\" data-bi-aN=\"margin-callout\" data-bi-cN=\"What advantages do belief state transformers offer over traditional GPT-style models in planning and decision-making tasks?\">\n\t\t\t\tWhat advantages do belief state transformers offer over traditional GPT-style models in planning and decision-making tasks?&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n\n\n<div class=\"wp-block-msr-show-more\">\n\t<div class=\"bg-neutral-100 p-5\">\n\t\t<div class=\"show-more-show-less\" data-mount=\"show-more-show-less\">\n\t\t\t<div>\n\t\t\t\t<span>\n\t\t\t\t\t\n\n<h3 class=\"wp-block-heading\" id=\"transcript-lightning-talk\">Transcript: Lightning Talk<\/h3>\n\n\n\n<p><strong>Belief state transformers<\/strong><\/p>\n\n\n\n<p><strong>John Langford<\/strong>, Partner Research Manager, Microsoft Research AI Frontiers<\/p>\n\n\n\n<p>This talk showcases a new transformer architecture that generates compact belief states for goal-conditioned planning, enhancing planning algorithms&#8217; efficiency and effectiveness.<\/p>\n\n\n\n<p>Microsoft Research Forum, February 25, 2025<\/p>\n\n\n\n\t\t\t\t<\/span>\n\t\t\t\t<span id=\"show-more-show-less-toggle-1\" class=\"show-more-show-less-toggleable-content\">\n\t\t\t\t\t\n\n\n\n<p><strong>FRIEDERIKE NIEDTNER, Principal Technical Research Program Manager, Microsoft Research AI Frontiers: <\/strong>Transformer models have brought us a revolution in language modeling with their capability to generate impressive language with many emergent properties. At the same time, LLMs have a number of weaknesses, one being that they are not very good at evaluating their own output. Let&#8217;s hear how the new Belief State Transformer architecture unlocks new abilities by combining a standard GPT-style architecture of a forward encoder for token prediction with an additional backward encoder.<\/p>\n\n\n\n<p><strong>JOHN LANGFORD<\/strong>: I&#8217;m John Langford. I&#8217;d like to tell you about <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/learning-to-achieve-goals-with-belief-state-transformers\/\">belief state transformers<\/a>, which is a new paper we have in archives, and which is also accepted at ICLR [International Conference on Learning Representations]. There are many coauthors on this paper. I&#8217;d like to thank them, particularly Edward, who did much of the work here.&nbsp;&nbsp;<\/p>\n\n\n\n<p>To start with, let&#8217;s talk about standard GPT-style transformers. In standard GPT style transformers, you have a sequence of symbols which are going into a forward encoder, and then the forward encoder outputs some information to the output head, and then the output head predicts the final token. So, this is a straightforward approach and yet amazingly powerful. It&#8217;s kind of the key backbone behind GPT-4 and other language models.&nbsp;&nbsp;<\/p>\n\n\n\n<p>For the purposes of research, though, we need to have something to think about, to complain about, and I&#8217;m going to complain about self-evaluation. Often these language models can&#8217;t be used to evaluate their own output too well, because the generation of the next token is done by exactly the mechanism you would use to evaluate it in that output head. So, this is kind of like grading yourself, and like grading yourself you can miss things that an independent grader would actually see pretty well.&nbsp;<\/p>\n\n\n\n<p>Right, so a belief state transformer changes the architecture. And so, it&#8217;s taking two transformers and grafting them together. One of them is going to be the standard forward encoder on the prefix. And then we&#8217;re also going to have another transformer, which is a backward encoder on the suffix. These are both going to put out some information, which goes to the output head. And the output head is going to predict the next token and the previous token. So, it&#8217;s the next token of the prefix and the previous token of the suffix. Something to worry about with these transformers is the computation. So, these are transformers obviously doing more computation. But it turns out that this \u201cmore computation\u201d is only in a constant factor of more computation.&nbsp;<\/p>\n\n\n\n<p>And the key observation here is that in the forward encoder, just doing the attention, what you&#8217;re going to use in the GPT-style transformer, is already order N-squared [N<sup>2<\/sup>]. Every token looks at every previous token in order to figure out what information is necessary to predict the next token. In the belief state transformer, that happens twice. You have two different transformers, each with their own attention, and so you pay a factor of two.&nbsp;&nbsp;<\/p>\n\n\n\n<p>And then, in addition, you&#8217;re going to pay because the number of times you evaluate the head, the output head, is order n squared because there are order N-squared prefix\/suffix pairs. So, there&#8217;s a constant factor increasing computation, which is problematic, but it&#8217;s not like the end of the world. You can subsample or things like that. And what you get in return is order N-squared gradients rather than order N gradients.&nbsp;<\/p>\n\n\n\n<p>In a standard GPT-style transformer, you only have order N gradients because you only have order N symbols, and you get one gradient per symbol. Here you get order N-squared gradients because you have order N-squared prefix\/suffix pairs. That means there&#8217;s many more ways to get information out of a sequence. And that unlocks the possibility of learning new things that were previously unlearnable.&nbsp;<\/p>\n\n\n\n<p>Okay, so now let&#8217;s go on to the belief state. Why are we talking about a belief state when we say belief state transformer. Well, it turns out you can prove a theorem. And this theorem says that the output of the forward encoder is a belief state for the prefix. So what that means is that the output of the forward encoder will converge to all the information necessary to predict the future. So that\u2019s all symbols after the prefix. So, that ability to create a compact belief state is new with belief state transformers, something that previously we only really knew how to do with state space machines.&nbsp;&nbsp;<\/p>\n\n\n\n<p>Okay, so let&#8217;s try this out. Looking at Tiny Stories. Tiny Stories is dataset where you have a bunch of children stories, which are generated by GPT-4.&nbsp;<\/p>\n\n\n\n<p>We&#8217;re going to feed a prefix and a suffix into our system, and it&#8217;s going to fill in the middle, which is what happens in blue. And then for a baseline, we&#8217;re going to compare the fill-in-the-middle approach to using GPT-style transformers. So the way the fill-in-the-middle approach works with GPT-style transformers is you take the prefix, and then you add the suffix, and then you just predict the tokens after that.&nbsp;<\/p>\n\n\n\n<p>So that works reasonably well. This is very commonly used. And now if we have these two different approaches the question is how do we actually value these different approaches? Which one is better? So, the way we&#8217;re going to judge this is we&#8217;re going to ask GPT-4 which is better in various ways: syntax, style, and so forth. And then we&#8217;ll ask it for a summary judgment, which is a standard technique.&nbsp;<\/p>\n\n\n\n<p>We looked at what it was doing, and it seemed very reasonable. And in doing this, we end up with the belief state transformer winning about a factor of three more often than the GPT-style transformer. So that&#8217;s huge. It&#8217;s so huge that you really want to understand why. And it seems like the key here is self-evaluation. So, under the hood, we&#8217;re actually running each of these, say 120 times, using a beam search. The code for that is on the right. So, given the beam search, you have several different possible completions. And now how do you choose which completion to actually use? Because you have to pick one of these. You&#8217;re trying to pick a completion. And for the GPT-style transformer, there&#8217;s only one way to really do this. The way is you take the next head, and you use it as a probability function, and you look at the probability of the sequence of tokens which is produced.&nbsp;&nbsp;<\/p>\n\n\n\n<p>That works reasonably well. It actually does improve picking out a high-probability sequence of tokens versus a lower probability sequence of tokens. But it&#8217;s not as much as you get with the belief state transformer. And the reason why is the self-grading issue that I was talking about earlier. There&#8217;s many ways that a system could be blind to its own mistakes. With the belief state transformer, though, you have another option, because the next head can instead condition on the generated data and run over the suffix in order to value the generated data.&nbsp;<\/p>\n\n\n\n<p>So, that ability to condition on generation, rather than evaluate the generation, ends up being amazingly useful in terms of giving you a more honest valuation of the generated text. All right, so just to summarize, we have this belief state transformer. This learns a compact belief state, which is a new thing in transformers. It gives us a way to have a simple set of values, which summarize all information we need to predict the future.&nbsp;<\/p>\n\n\n\n<p>And this seems to provide a very strong form of self-evaluation, which is potentially very useful in many situations where you&#8217;re trying to use test-time compute, or even using test-time compute to further create training data. So, this is more in the paper. There&#8217;s some other things that you can do with transformer that are kind of new.&nbsp;<\/p>\n\n\n\n<p>I think the biggest question in my mind is what happens when you scale this up? And, of course, we&#8217;re working on that. That&#8217;s one of the great things about being in MSR [Microsoft Research]. They have some GPUs to scale this up to much larger datasets. So, stay tuned. And, thank you.&nbsp;<\/p>\n\n\t\t\t\t<\/span>\n\t\t\t<\/div>\n\t\t\t<button\n\t\t\t\tclass=\"action-trigger glyph-prepend mt-2 mb-0 show-more-show-less-toggle\"\n\t\t\t\taria-expanded=\"false\"\n\t\t\t\tdata-show-less-text=\"Show less\"\n\t\t\t\ttype=\"button\"\n\t\t\t\taria-controls=\"show-more-show-less-toggle-1\"\n\t\t\t\taria-label=\"Show more content\"\n\t\t\t\tdata-alternate-aria-label=\"Show less content\">\n\t\t\t\tShow more\t\t\t<\/button>\n\t\t<\/div>\n\t<\/div>\n<\/div>\n\n\n\n<h2 class=\"wp-block-heading alignwide\" id=\"related-resources\">Related resources<\/h2>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Research Lab<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/lab\/ai-frontiers\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"AI Frontiers\" data-bi-aN=\"citation\" data-bi-cN=\"AI Frontiers\">\n\t\t\t\tAI Frontiers&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/learning-to-achieve-goals-with-belief-state-transformers\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Learning to Achieve Goals with Belief State Transformers\" data-bi-aN=\"citation\" data-bi-cN=\"Learning to Achieve Goals with Belief State Transformers\">\n\t\t\t\tLearning to Achieve Goals with Belief State Transformers&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\"><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>John Langford talks about a new transformer architecture that generates compact belief states for goal-conditioned planning, enhancing planning algorithms&#8217; efficiency and effectiveness.<\/p>\n","protected":false},"author":38004,"featured_media":1130985,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-content-parent":1123713,"footnotes":""},"research-area":[13556],"msr-locale":[268875],"msr-post-option":[269148,269142],"class_list":["post-1123773","msr-blog-post","type-msr-blog-post","status-publish","has-post-thumbnail","hentry","msr-research-area-artificial-intelligence","msr-locale-en_us","msr-post-option-approved-for-river","msr-post-option-include-in-river"],"msr_assoc_parent":{"id":1123713,"type":"story"},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1123773","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-blog-post"}],"author":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/users\/38004"}],"version-history":[{"count":8,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1123773\/revisions"}],"predecessor-version":[{"id":1132170,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1123773\/revisions\/1132170"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media\/1130985"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=1123773"}],"wp:term":[{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=1123773"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=1123773"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=1123773"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}