{"id":1011558,"date":"2024-03-08T16:49:10","date_gmt":"2024-03-09T00:49:10","guid":{"rendered":"https:\/\/www.microsoft.com\/en-us\/research\/?post_type=msr-blog-post&#038;p=1011558"},"modified":"2024-03-08T16:49:12","modified_gmt":"2024-03-09T00:49:12","slug":"the-metacognitive-demands-and-opportunities-of-generative-ai","status":"publish","type":"msr-blog-post","link":"https:\/\/www.microsoft.com\/en-us\/research\/articles\/the-metacognitive-demands-and-opportunities-of-generative-ai\/","title":{"rendered":"The Metacognitive Demands and Opportunities of Generative AI"},"content":{"rendered":"\n<p class=\"has-purple-color has-text-color has-link-color wp-elements-4c9e3eb45b4a4e8cf664c3d0c5ba8257\"><em>Presented by&nbsp;<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/t-levt\/\">Lev Tankelevitch<\/a>&nbsp;at&nbsp;<strong>Microsoft Research Forum, March 2024<\/strong><\/em><\/p>\n\n\n\n<div class=\"wp-block-media-text has-vertical-margin-none  has-vertical-padding-none  is-stacked-on-mobile has-white-background-color has-background\" style=\"grid-template-columns:25% auto\" data-bi-an=\"media-text\"><figure class=\"wp-block-media-text__media\"><img loading=\"lazy\" decoding=\"async\" width=\"360\" height=\"360\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Lev-Tankelevitch_360x360.jpg\" alt=\"Lev Tankelevitch\" class=\"wp-image-1012581 size-full\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Lev-Tankelevitch_360x360.jpg 360w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Lev-Tankelevitch_360x360-300x300.jpg 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Lev-Tankelevitch_360x360-150x150.jpg 150w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Lev-Tankelevitch_360x360-180x180.jpg 180w\" sizes=\"(max-width: 360px) 100vw, 360px\" \/><\/figure><div class=\"wp-block-media-text__content\" data-bi-an=\"media-text\">\n<blockquote class=\"wp-block-quote is-style-spectrum is-layout-flow wp-block-quote-is-layout-flow\">\n<p>\u201cWe believe that a metacognitive perspective can really help us analyze, measure, and evaluate the usability challenges of generative AI. And it can help us design generative AI systems that can augment human agency and workflows.\u201d<\/p>\n<cite><em>\u2013<\/em> Lev Tankelevitch, Senior Behavioral Science Researcher, Microsoft Research Cambridge<\/cite><\/blockquote>\n<\/div><\/div>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"The Metacognitive Demands and Opportunities of Generative AI\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube-nocookie.com\/embed\/cyfS6APUbHU?feature=oembed&rel=0\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<div class=\"annotations \" data-bi-aN=\"margin-callout\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 annotations__list--right\">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Explain%20how%20metacognition%20can%20inform%20Generative%20AI%20usability.\" target=\"_blank\" aria-label=\"Explain how metacognition can inform Generative AI usability.\" data-bi-type=\"annotated-link\" data-bi-cN=\"Explain how metacognition can inform Generative AI usability.\" class=\"annotations__list-thumbnail\" >\n\t\t\t\t\t<img loading=\"lazy\" decoding=\"async\" width=\"172\" height=\"96\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png\" class=\"mb-2\" alt=\"Ask Microsoft research copilot experience\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png 240w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-300x169.png 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1024x576.png 1024w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-768x432.png 768w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1066x600.png 1066w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-655x368.png 655w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-343x193.png 343w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-640x360.png 640w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-960x540.png 960w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1280x720.png 1280w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo.png 1400w\" sizes=\"(max-width: 172px) 100vw, 172px\" \/>\t\t\t\t<\/a>\n\t\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Microsoft research copilot experience<\/span>\n\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Explain%20how%20metacognition%20can%20inform%20Generative%20AI%20usability.\" target=\"_blank\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Explain how metacognition can inform Generative AI usability.\" data-bi-aN=\"margin-callout\" data-bi-cN=\"Explain how metacognition can inform Generative AI usability.\">\n\t\t\t\tExplain how metacognition can inform Generative AI usability.&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n\n\n<div class=\"wp-block-msr-show-more\">\n\t<div class=\"bg-neutral-100 p-5\">\n\t\t<div class=\"show-more-show-less\" data-mount=\"show-more-show-less\">\n\t\t\t<div>\n\t\t\t\t<span>\n\t\t\t\t\t\n\n<h3 class=\"wp-block-heading\" id=\"transcript-lightning-talk-2\">Transcript: Lightning Talk 2<\/h3>\n\n\n\n<p><strong>The metacognitive demands and opportunities of generative AI<\/strong><\/p>\n\n\n\n<p><strong>Lev Tankelevitch<\/strong>, Senior Behavioral Science Researcher, Microsoft Research Cambridge<\/p>\n\n\n\n<p>Lev Tankelevitch explores how metacognition\u2014the psychological capacity to monitor and regulate one\u2019s cognitive processes\u2014provides a valuable perspective for comprehending and addressing the usability challenges of generative AI systems around prompting, assessing and relying on outputs, and workflow optimization.<\/p>\n\n\n\n<p>Microsoft Research Forum, March 5, 2024<\/p>\n\n\n\n\t\t\t\t<\/span>\n\t\t\t\t<span id=\"show-more-show-less-toggle-1\" class=\"show-more-show-less-toggleable-content\">\n\t\t\t\t\t\n\n\n\n<p><strong>LEV TANKELEVITCH: <\/strong>My name is Lev. I\u2019m a researcher in the Collaborative Intelligence team in Microsoft Research Cambridge, UK, and today I\u2019ll be talking about what we\u2019re calling the metacognitive demands and opportunities of generative AI. So we know that AI has tremendous potential to transform personal and professional work. But as we show in our recent paper, a lot of usability challenges remain\u2014from crafting the right prompts to evaluating and relying on outputs to integrating AI into our daily workflows. And [what] we propose in a recent paper is that metacognition offers a powerful framework to understand and design for these usability challenges.&nbsp;&nbsp;<\/p>\n\n\n\n<p>So metacognition is thinking <em>about<\/em> thinking and includes things like self-awareness, so our ability to be aware of our own goals, knowledge, abilities, and strategies; our confidence and its adjustment, so this is our ability to maintain an appropriate level of confidence in our knowledge and abilities and adjust that as new information comes in; task decomposition, our ability to take a cognitive task or goal and break it down into subtasks and address them in turn; and metacognitive flexibility, so our ability to recognize when a cognitive strategy <em>isn&#8217;t<\/em> working and adapt it accordingly. Let me walk you through a simple example workflow.<\/p>\n\n\n\n<p>So let\u2019s say you decided to ask an AI system to help you in crafting an email. So in the beginning, you might have to craft a prompt. And so you might ask yourself, what am I trying to convey with this email? Perhaps I need to summarize <em>x<\/em>, clarify <em>y<\/em>, or conclude <em>z<\/em>\u2014all in the correct tone. You might then get an output and then need to evaluate that. And then you might ask yourself, well, how can I make sense of this output? In the case of an email example, it\u2019s pretty straightforward. But what if you\u2019re working with a programming language that you\u2019re less familiar with? You might then need to iterate on your prompt. And so then you might ask yourself, well, how does it relate to my ability to craft the right prompt versus the system\u2019s performance in a given task or domain?&nbsp;&nbsp;<\/p>\n\n\n\n<p>And now if you zoom out a little bit, there are these questions around what we\u2019re calling <em>automation strategy<\/em>. So this is whether, when, and how you can apply AI to your workflows. So here you might ask yourself, is trying generative AI worth my time versus doing a task manually? And how confident am I that I can actually complete a task manually or learn AI effectively to help me do it? And then if I do decide to rely on AI on my workflows, how do I actually integrate it into my workflows most effectively? And so what we\u2019re proposing is that all these questions really reflect the metacognitive demands that generative AI systems impose on users as they interact with these systems. So, for example, at the prompt formulation stage, this involves self-awareness of task goals. So knowing exactly what you want to achieve and break that down into subgoals and subtasks and then verbalize that explicitly for an effective prompt. At the output evaluation stage, it involves well-adjusted confidence in your ability to actually evaluate that output. And so that means disentangling your confidence in the domain you\u2019re working with from the system\u2019s performance in that task or domain.&nbsp;&nbsp;<\/p>\n\n\n\n<p>In the prompt iteration stage, it involves well-adjusted confidence in your prompting ability, so this is about disentangling your ability to craft an effective prompt from the system&#8217;s performance in that task or domain, and metacognitive flexibility, which is about recognizing when a prompting strategy isn\u2019t working and then adjusting it accordingly. In the automation strategy level, this is about self-awareness of the applicability and impact of AI on your workflows and well-adjusted confidence in your ability to complete a task manually or learn generative AI systems effectively to actually help you do that. And then finally, it requires metacognitive flexibility in actually recognizing when your workflow with AI isn\u2019t working effectively and adapting that accordingly.<\/p>\n\n\n\n<p>So beyond reframing these usability challenges through the perspective of metacognition, we know from psychology research that metacognition is both measurable and teachable. And so we can now think about how we can design systems that actually support people\u2019s metacognition as they interact with them. So, for example, you can imagine systems that support people in planning complex tasks. So let\u2019s say you\u2019ve decided to ask an AI system to help you craft an email. It might actually break that task down for you and remind you that certain types of content are more common in such emails and actually proactively prompt you to fill that content in. It might also make you aware of the fact that there\u2019s a certain tone or length that you might want to have for this email. And so in this way, it, sort of, breaks the task down for you and actually improves your self-awareness about different aspects of your task.&nbsp;&nbsp;<\/p>\n\n\n\n<p>Similarly, we can imagine systems that support people in reflecting on their own cognition. So let\u2019s say you\u2019ve asked the system to help you craft a proposal based on a previous document. Now a smart system that knows in the past you\u2019ve had to edit this output quite extensively might let you know that you should specify an outline or other details and provide you with examples so that you can save time later on. Similarly, at the output evaluation stage, you can imagine how such an approach can augment AI explanations. So this is work done by the <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/project\/calc-intelligence\/\">Calc Intelligence<\/a> team here at Microsoft Research, and it shows a system that can help users complete tasks in spreadsheets. And it shows a step-by-step breakdown of the approach that it took to complete that task. So you can imagine a system that proactively probes users about different steps and their uncertainty around those steps and then tailors explanations effectively to that user\u2019s uncertainty.&nbsp;&nbsp;<\/p>\n\n\n\n<p>So in sum, we believe that a metacognitive perspective can really help us analyze, measure, and evaluate the usability challenges of generative AI. <em>And<\/em> it can help us design generative AI systems that can augment human agency and workflows. For more details, I encourage you to check out the full paper, and I thank you for your time.<\/p>\n\n\t\t\t\t<\/span>\n\t\t\t<\/div>\n\t\t\t<button\n\t\t\t\tclass=\"action-trigger glyph-prepend mt-2 mb-0 show-more-show-less-toggle\"\n\t\t\t\taria-expanded=\"false\"\n\t\t\t\tdata-show-less-text=\"Show less\"\n\t\t\t\ttype=\"button\"\n\t\t\t\taria-controls=\"show-more-show-less-toggle-1\"\n\t\t\t\taria-label=\"Show more content\"\n\t\t\t\tdata-alternate-aria-label=\"Show less content\">\n\t\t\t\tShow more\t\t\t<\/button>\n\t\t<\/div>\n\t<\/div>\n<\/div>\n\n\n\n<div style=\"padding-bottom:64px; padding-top:64px\" class=\"wp-block-msr-immersive-section alignfull row wp-block-msr-immersive-section\">\n\t\n\t<div class=\"container\">\n\t\t<div class=\"wp-block-msr-immersive-section__inner\">\n\t\t\t<div class=\"wp-block-columns is-layout-flex wp-container-core-columns-is-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:33.33%\">\n<hr class=\"wp-block-separator has-text-color has-purple-color has-alpha-channel-opacity has-purple-background-color has-background\"\/>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"the-metacognitive-demands-and-opportunities-of-generative-ai\">The Metacognitive Demands and Opportunities of Generative AI<\/h2>\n\n\n\n<p>By <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/t-levt\/\">Lev Tankelevitch<\/a><\/p>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:66.66%\">\n<p><strong>Generative AI (GenAI) systems offer unprecedented opportunities for transforming professional and personal work.<\/strong> This potential stems from a <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/www.jair.org\/index.php\/jair\/article\/view\/14157\" target=\"_blank\" rel=\"noreferrer noopener\">unique combination<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> of generative AI\u2019s <em>model flexibility<\/em>, in that systems can accommodate a wide range of information in prompts and outputs; <em>generality<\/em>, in that systems are applicable to a wide range of tasks; and <em>originality<\/em>, in that systems can generate novel content. However, these properties are a double-edged sword, in that they also pose usability challenges for people working with GenAI systems. Studies show that people find it difficult to craft effective prompts, evaluate and rely on AI outputs, and optimize their workflows with GenAI. In <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/arxiv.org\/abs\/2312.10893\" target=\"_blank\" rel=\"noreferrer noopener\">recent work<span class=\"sr-only\"> (opens in new tab)<\/span><\/a>, we propose that <em>metacognition<\/em>\u2014the psychological ability to monitor and control one\u2019s thoughts and behavior\u2014 offers a valuable lens through which to understand and design for these usability challenges.<\/p>\n<\/div>\n<\/div>\n\n\n\n<p>Current GenAI systems impose multiple metacognitive demands on users. A useful analogy for how people work with GenAI systems is that of a manager delegating tasks to a team. A manager needs to clearly understand and formulate their goals, break down those goals into communicable tasks, confidently assess the quality of the team\u2019s output, and adjust plans accordingly along the way. Moreover, a manager needs to decide whether, when, and how to delegate tasks in the first place. All these responsibilities involve the metacognitive monitoring and control of one\u2019s thought processes and behavior (i.e., cognition). Working successfully with GenAI systems requires these same abilities.<\/p>\n\n\n\n<p>Prompting is the first challenge people face when interacting with GenAI systems, according to <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/arxiv.org\/pdf\/2208.06213\" target=\"_blank\" rel=\"noreferrer noopener\">multiple<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3544548.3581388\" target=\"_blank\" rel=\"noreferrer noopener\">studies<span class=\"sr-only\"> (opens in new tab)<\/span><\/a>. With a manual task, such as drafting an email, many implicit goals and intentions can remain so without ever being verbalized. For example, one might implicitly know to adopt a certain tone when writing to a senior colleague, or that a given draft first requires summarizing certain content and then pulling out key points for discussion. In contrast, all these details need to be explicitly specified in order for GenAI systems to execute tasks effectively. This requires people to be aware of their goals, have the ability to break down tasks into sub-tasks, and to verbalize all of this within a set of prompts\u2014abilities comprising metacognitive monitoring and control. After submitting a prompt, a common step is assessing the output (i.e., determining whether the system achieved what was intended) and iterating on the prompt. If the result is not satisfactory, a key challenge for users is disentangling the role of their prompting ability from that of the system\u2019s capabilities. A poor result can arise due to a poorly worded prompt, the system\u2019s settings, the system\u2019s limitations on a given task, or an \u201cunlucky draw\u201d given the probabilistic nature of many GenAI systems. The range of possible explanations make it difficult for people to appropriately adjust their confidence in their prompting ability, a key form of metacognitive monitoring\u2014much like a manager who can\u2019t figure out whether their instructions are unclear or if their team is underperforming for other reasons. A subsequent challenge for users is being flexible enough to change prompting strategies as needed, whether that means rewording their most recent prompt, breaking up their task into further sub-tasks, or taking another approach.<\/p>\n\n\n\n<p>Putting prompting aside, assessing the system\u2019s output poses further challenges for people in terms of deciding whether to rely on it. This is particularly true when people use GenAI in a domain in which they have <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/arxiv.org\/pdf\/2304.02491\" target=\"_blank\" rel=\"noreferrer noopener\">little expertise<span class=\"sr-only\"> (opens in new tab)<\/span><\/a>, such as a new programming language. Again, the challenge here is to appropriately adjust one\u2019s confidence, albeit now in the domain itself and the ability to assess the output. This is critical given the risk of incorrect or incoherent results, or other errors that GenAI systems can produce. Alongside the many possible explanations for a given output, this is further exacerbated by the speed and ease with which GenAI systems can produce extensive outputs, including entire presentations, reports, or software. This is akin to a manager with a prolific team that they cannot fully trust. Maintaining an appropriate level of confidence in assessing GenAI output is important, because this may influence the extent of over- or under-reliance on this technology.<\/p>\n\n\n\n<p>Zooming out from individual interactions with GenAI systems, there is a broader question of how people can determine <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/advait.org\/files\/sarkar_2023_ai_knowledge_work.pdf\" target=\"_blank\" rel=\"noreferrer noopener\">whether, when, and how they should use GenAI<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> for tasks within their workflows. Determining this \u201cautomation strategy\u201d requires awareness of, and an appropriate level of confidence in, the applicability and potential impact of using GenAI within a workflow. For example, people need to recognise when relying on GenAI is more productive than doing a task manually, or vice versa. Over time, people also need the flexibility to change strategies as they assess their workflows.<\/p>\n\n\n\n<p>Rather than imposing metacognitive demands on users, GenAI\u2019s model flexibility, generality, and originality also present many opportunities for improving people\u2019s metacognition\u2014that is, their self-awareness, adjustment of confidence, and flexibility in their cognition. An exciting area for exploration is designing systems that can proactively support users in planning tasks with GenAI. For example, systems can transform people\u2019s initial high-level task into <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/arxiv.org\/abs\/2110.01691\" target=\"_blank\" rel=\"noreferrer noopener\">a series of sub-tasks<span class=\"sr-only\"> (opens in new tab)<\/span><\/a>, clarifying their goals in the process, and helping them craft effective prompts. At the output assessment stage, systems can also proactively <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3544548.3580999\" target=\"_blank\" rel=\"noreferrer noopener\">help people reflect<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> on the output, their confidence in it, and any uncertainties they may have, either in the GenAI interaction or in the domain itself. Indeed, by adapting information to people\u2019s level of confidence and areas of uncertainty, GenAI systems could help augment <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3544548.3580817\" target=\"_blank\" rel=\"noreferrer noopener\">explanations<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> for their own outputs in an interactive manner. At a broader level, GenAI systems can proactively support people in reflecting on and, in turn, optimizing their workflows as they incorporate GenAI into their tasks. In a way, it\u2019s as if each manager had a personal coach and mentor that supported them as they learned to manage their team.<\/p>\n\n\n\n<p>As we offload more of our cognitive tasks (e.g., ideation, synthesis, writing) to GenAI systems, it becomes increasingly important to understand how we monitor and control our cognition. This is precisely where the perspective of metacognition fits in. Beyond helping us re-frame the key usability challenges emerging in human-GenAI interaction, metacognition can also inspire novel system designs that augment our cognition, improving our self-awareness, confidence adjustment, and flexibility as we work with GenAI systems. As hinted here, there is a lot more work to do in this space.<\/p>\n\n\n\n<p><em>The thinking around metacognition and GenAI described here is a Microsoft Research project, with key collaborators that include Viktor Kewenig, Auste Simkute, Ava Scott, <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/advait\/\">Advait Sarkar<\/a>, <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/asellen\/\">Abi Sellen<\/a>, and <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/serintel\/\">Sean Rintel<\/a>.<\/em><\/p>\n\n\n\n<hr class=\"wp-block-separator has-text-color has-white-color has-alpha-channel-opacity has-white-background-color has-background is-style-end-mark\"\/>\t\t<\/div>\n\t<\/div>\n\n\t<\/div>\n\n\n\n<h2 class=\"wp-block-heading alignwide\" id=\"related-resources\">Related resources<\/h2>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-2 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Research Lab<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/lab\/microsoft-research-cambridge\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Microsoft Research Lab \u2013 Cambridge\" data-bi-aN=\"citation\" data-bi-cN=\"Microsoft Research Lab \u2013 Cambridge\">\n\t\t\t\tMicrosoft Research Lab \u2013 Cambridge&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Group<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/theme\/collaborative-intelligence\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Collaborative Intelligence\" data-bi-aN=\"citation\" data-bi-cN=\"Collaborative Intelligence\">\n\t\t\t\tCollaborative Intelligence&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Project<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/project\/calc-intelligence\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Calc Intelligence\" data-bi-aN=\"citation\" data-bi-cN=\"Calc Intelligence\">\n\t\t\t\tCalc Intelligence&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-3 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Podcast<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/podcast\/abstracts-february-29-2024\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Abstracts: February 29, 2024\" data-bi-aN=\"citation\" data-bi-cN=\"Abstracts: February 29, 2024\">\n\t\t\t\tAbstracts: February 29, 2024&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/the-metacognitive-demands-and-opportunities-of-generative-ai\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"The Metacognitive Demands and Opportunities of Generative AI\" data-bi-aN=\"citation\" data-bi-cN=\"The Metacognitive Demands and Opportunities of Generative AI\">\n\t\t\t\tThe Metacognitive Demands and Opportunities of Generative AI&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/what-it-wants-me-to-say-bridging-the-abstraction-gap-between-end-user-programmers-and-code-generating-large-language-models\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"\u201cWhat It Wants Me To Say\u201d: Bridging the Abstraction Gap Between End-User Programmers and Code-Generating Large Language Models\" data-bi-aN=\"citation\" data-bi-cN=\"\u201cWhat It Wants Me To Say\u201d: Bridging the Abstraction Gap Between End-User Programmers and Code-Generating Large Language Models\">\n\t\t\t\t\u201cWhat It Wants Me To Say\u201d: Bridging the Abstraction Gap Between End-User Programmers and Code-Generating Large Language Models&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>Lev Tankelevitch explores how metacognition\u2014the psychological capacity to monitor and regulate one&#8217;s cognitive processes\u2014provides a valuable perspective for comprehending and addressing the usability challenges of generative AI systems around prompting, assessing and relying on outputs, and workflow optimization.<\/p>\n","protected":false},"author":42735,"featured_media":1012578,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-content-parent":1009965,"footnotes":""},"research-area":[],"msr-locale":[268875],"msr-post-option":[],"class_list":["post-1011558","msr-blog-post","type-msr-blog-post","status-publish","has-post-thumbnail","hentry","msr-locale-en_us"],"msr_assoc_parent":{"id":1009965,"type":"story"},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011558"}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-blog-post"}],"author":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/users\/42735"}],"version-history":[{"count":11,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011558\/revisions"}],"predecessor-version":[{"id":1013547,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011558\/revisions\/1013547"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media\/1012578"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=1011558"}],"wp:term":[{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=1011558"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=1011558"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=1011558"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}