{"id":999732,"date":"2024-01-30T05:18:54","date_gmt":"2024-01-30T13:18:54","guid":{"rendered":"https:\/\/www.microsoft.com\/en-us\/research\/?post_type=msr-blog-post&#038;p=999732"},"modified":"2024-06-10T09:54:15","modified_gmt":"2024-06-10T16:54:15","slug":"evaluation-and-understanding-of-foundation-models","status":"publish","type":"msr-blog-post","link":"https:\/\/www.microsoft.com\/en-us\/research\/articles\/evaluation-and-understanding-of-foundation-models\/","title":{"rendered":"Evaluation and Understanding of Foundation Models"},"content":{"rendered":"\n<p class=\"has-purple-color has-text-color has-link-color wp-elements-0021f1cb6ba8e2f1046cab93ac68b18d\"><em>Presented by <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/benushi\/\" target=\"_blank\" rel=\"noreferrer noopener\">Besmira Nushi<\/a> at <strong>Microsoft Research Forum, January 2024<\/strong><\/em><\/p>\n\n\n\n<div class=\"wp-block-media-text has-vertical-margin-none  has-vertical-padding-none  is-stacked-on-mobile has-white-background-color has-background\" style=\"grid-template-columns:25% auto\" data-bi-an=\"media-text\"><figure class=\"wp-block-media-text__media\"><img loading=\"lazy\" decoding=\"async\" width=\"360\" height=\"360\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2023\/02\/Besmira-Nushi_360x360.jpg\" alt=\"photo of Besmira Nushi smiling for the camera\" class=\"wp-image-918282 size-full\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2023\/02\/Besmira-Nushi_360x360.jpg 360w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2023\/02\/Besmira-Nushi_360x360-300x300.jpg 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2023\/02\/Besmira-Nushi_360x360-150x150.jpg 150w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2023\/02\/Besmira-Nushi_360x360-180x180.jpg 180w\" sizes=\"auto, (max-width: 360px) 100vw, 360px\" \/><\/figure><div class=\"wp-block-media-text__content\" data-bi-an=\"media-text\">\n<blockquote class=\"wp-block-quote is-style-spectrum is-layout-flow wp-block-quote-is-layout-flow\">\n<p>\u201cWe see model evaluation and understanding as a guide to AI innovation. Our work measures, informs, and accelerates model improvement and, at the same time, is a contribution that is useful to the scientific community for understanding and studying new forms and levels of intelligence.\u201d<\/p>\n<cite><em>\u2013<\/em> Besmira Nushi, Principal Researcher<\/cite><\/blockquote>\n<\/div><\/div>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"Evaluation and Understanding of Foundation Models\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube-nocookie.com\/embed\/p3QNY9AvNAM?feature=oembed&rel=0\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<div class=\"annotations \" data-bi-aN=\"margin-callout\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 annotations__list--right\">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Summarize%20the%20main%20three%20points%20of%20Besmira%27s%20talk\" target=\"_blank\" aria-label=\"Summarize the main three points of Besmira's talk\" data-bi-type=\"annotated-link\" data-bi-cN=\"Summarize the main three points of Besmira's talk\" class=\"annotations__list-thumbnail\" >\n\t\t\t\t\t<img loading=\"lazy\" decoding=\"async\" width=\"172\" height=\"96\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png\" class=\"mb-2\" alt=\"Ask Microsoft research copilot experience\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png 240w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-300x169.png 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1024x576.png 1024w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-768x432.png 768w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1066x600.png 1066w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-655x368.png 655w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-343x193.png 343w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-640x360.png 640w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-960x540.png 960w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1280x720.png 1280w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo.png 1400w\" sizes=\"auto, (max-width: 172px) 100vw, 172px\" \/>\t\t\t\t<\/a>\n\t\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Microsoft research copilot experience<\/span>\n\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Summarize%20the%20main%20three%20points%20of%20Besmira%27s%20talk\" target=\"_blank\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Summarize the main three points of Besmira's talk\" data-bi-aN=\"margin-callout\" data-bi-cN=\"Summarize the main three points of Besmira's talk\">\n\t\t\t\tSummarize the main three points of Besmira's talk&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n\n\n<div class=\"wp-block-msr-show-more\">\n\t<div class=\"bg-neutral-100 p-5\">\n\t\t<div class=\"show-more-show-less\" data-mount=\"show-more-show-less\">\n\t\t\t<div>\n\t\t\t\t<span>\n\t\t\t\t\t\n\n<h3 class=\"wp-block-heading\" id=\"transcript\">Transcript<\/h3>\n\n\n\n<p><strong>Besmira Nushi<\/strong>, Principal Researcher, Microsoft Research AI Frontiers&nbsp;<\/p>\n\n\n\n<p>Besmira Nushi summarizes timely challenges and ongoing work on evaluating and in-depth understanding of large foundation models as well as agent platforms built upon such models.&nbsp;<\/p>\n\n\n\n<p><em>Microsoft Research Forum,&nbsp;January 30, 2024<\/em><\/p>\n\n\n\n<p><strong>BESMIRA NUSHI:&nbsp;<\/strong>Hi, everyone. My name is Besmira Nushi, and together with my colleagues at Microsoft Research, I work on evaluating and understanding foundation models. In our team, we see model evaluation and understanding as a guide to AI innovation. Our work measures, informs, and accelerates model improvement and, at the same time, is a contribution that is useful to the scientific community for understanding and studying new forms and levels of intelligence.<\/p>\n\n\n\n\t\t\t\t<\/span>\n\t\t\t\t<span id=\"show-more-show-less-toggle-1\" class=\"show-more-show-less-toggleable-content\">\n\t\t\t\t\t\n\n\n\n<p>But evaluation is hard, and new generative tasks are posing new challenges in evaluation and understanding. For example, it has become really difficult to scale up evaluation for long, open-ended, and generative outputs. At the same time, for emergent abilities, very often some benchmarks do not exist and often we have to create them from scratch. And even when they exist, they may be saturated or leaked into training datasets. In other cases, factors like prompt variability and model updates may be just as important as the quality of the model that is being tested in the first place. When it comes to end-to-end and interactive scenarios, other aspects of model behavior may get in the way and may interfere with task completion and user satisfaction. And finally, there exists a gap between evaluation and model improvement.&nbsp;<\/p>\n\n\n\n<p>In our work, we really see this as just the first step towards understanding new failure modes and new architectures through data and model understanding. So in Microsoft Research, when we address these challenges, we look at four important pillars. First, we build novel benchmarks and evaluation workflows. Second, we perform and put a focus on interactive and multi-agent systems evaluation. And in everything we do, in every report that we write, we put responsible AI at the center of testing and evaluation to understand the impact of our technology on society. Finally, to bridge the gap between evaluation and improvement, we pursue efforts in data and model understanding.&nbsp;&nbsp;<\/p>\n\n\n\n<p>But let&#8217;s look at some examples. Recently, in the benchmark space, we released <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/kitab-evaluating-llms-on-constraint-satisfaction-for-information-retrieval\/\">KITAB<\/a>. KITAB is a novel benchmark and dataset for testing constraint satisfaction capabilities for information retrieval queries that have certain user specifications in terms of constraints. And when we tested recent state-of-the-art models with this benchmark, we noticed that only in 50 percent of the cases these models are able to satisfy user constraints.<\/p>\n\n\n\n<p>And similarly, in the multimodal space, Microsoft Research just released <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/holoassist-an-egocentric-human-interaction-dataset-for-interactive-ai-assistants-in-the-real-world\/\">HoloAssist<span class=\"sr-only\"> (opens in new tab)<\/span><\/a>. HoloAssist is a testbed with extensive amounts of data that come from recording and understanding how people perform tasks in the real and physical world. And this provides us with an invaluable amount of resources in terms of evaluation for understanding and measuring how the new models are going to assist people in things like task completion and mistake correction. In the responsible AI area, <a class=\"msr-external-link glyph-append glyph-append-open-in-new-tab glyph-append-xsmall\" href=\"https:\/\/github.com\/microsoft\/TOXIGEN\/\" target=\"_blank\" rel=\"noreferrer noopener\">ToxiGen<span class=\"sr-only\"> (opens in new tab)<\/span><\/a> is a new dataset that is designed to mention and to understand toxicity generation from language models. And it is able to measure harms that may be generated from such models across 13 different demographic groups.&nbsp;<\/p>\n\n\n\n<p>Similarly, in the multimodal space, we ran extensive evaluations to measure representational fairness and biases. For example, we tested several image generation models to see how they represent certain occupations, certain personality traits, and geographical locations. And we found that sometimes such models may present a major setback when it comes to representing different occupations if compared to real-world representation. For instance, in some cases, we see as low as 0 percent representation for certain demographic groups. &nbsp;<\/p>\n\n\n\n<p>Now when it comes to data on model understanding, often what we do is that we look back at architectural and model behavior patterns to see how they are tied to important and common errors in the space. For example, for the case of constraint satisfaction for user queries, we looked at factual errors, information fabrication and mapped them to important attention patterns. And we see that whenever factual errors occur, there are very weak attention patterns within the model that map to these errors. And this is an important finding that is going to inform our next steps in model improvement.&nbsp;<\/p>\n\n\n\n<p>So as we push the new frontiers in AI innovation, we are also just as excited about understanding and measuring that progress scientifically. And we hope that many of you are going to join us in that challenge. <\/p>\n\n\n\n<p>Thank you.<\/p>\n\n\t\t\t\t<\/span>\n\t\t\t<\/div>\n\t\t\t<button\n\t\t\t\tclass=\"action-trigger glyph-prepend mt-2 mb-0 show-more-show-less-toggle\"\n\t\t\t\taria-expanded=\"false\"\n\t\t\t\tdata-show-less-text=\"Show less\"\n\t\t\t\ttype=\"button\"\n\t\t\t\taria-controls=\"show-more-show-less-toggle-1\"\n\t\t\t\taria-label=\"Show more content\"\n\t\t\t\tdata-alternate-aria-label=\"Show less content\">\n\t\t\t\tShow more\t\t\t<\/button>\n\t\t<\/div>\n\t<\/div>\n<\/div>\n\n\n\n<h3 class=\"wp-block-heading alignwide\" id=\"related-resources\">Related resources<\/h3>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/kitab-evaluating-llms-on-constraint-satisfaction-for-information-retrieval\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"KITAB: Evaluating LLMs on Constraint Satisfaction for Information Retrieval\" data-bi-aN=\"citation\" data-bi-cN=\"KITAB: Evaluating LLMs on Constraint Satisfaction for Information Retrieval\">\n\t\t\t\tKITAB: Evaluating LLMs on Constraint Satisfaction for Information Retrieval&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Blog<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/blog\/holoassist-a-multimodal-dataset-for-next-gen-ai-copilots-for-the-physical-world\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"HoloAssist: A multimodal dataset for next-gen AI copilots for the physical world\" data-bi-aN=\"citation\" data-bi-cN=\"HoloAssist: A multimodal dataset for next-gen AI copilots for the physical world\">\n\t\t\t\tHoloAssist: A multimodal dataset for next-gen AI copilots for the physical world&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/toxigen-a-large-scale-machine-generated-dataset-for-adversarial-and-implicit-hate-speech-detection\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"ToxiGen: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection\" data-bi-aN=\"citation\" data-bi-cN=\"ToxiGen: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection\">\n\t\t\t\tToxiGen: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-2 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/social-biases-through-the-text-to-image-generation-lens\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Social Biases through the Text-to-Image Generation Lens\" data-bi-aN=\"citation\" data-bi-cN=\"Social Biases through the Text-to-Image Generation Lens\">\n\t\t\t\tSocial Biases through the Text-to-Image Generation Lens&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/attention-satisfies-a-constraint-satisfaction-lens-on-factual-errors-of-language-models\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Attention Satisfies: A Constraint-Satisfaction Lens on Factual Errors of Language Models\" data-bi-aN=\"citation\" data-bi-cN=\"Attention Satisfies: A Constraint-Satisfaction Lens on Factual Errors of Language Models\">\n\t\t\t\tAttention Satisfies: A Constraint-Satisfaction Lens on Factual Errors of Language Models&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Research Lab<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/lab\/microsoft-research-ai-for-science\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Microsoft Research AI for Science\" data-bi-aN=\"citation\" data-bi-cN=\"Microsoft Research AI for Science\">\n\t\t\t\tMicrosoft Research AI for Science&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>Besmira Nushi summarizes timely challenges and ongoing work on evaluating and in-depth understanding of large foundation models as well as agent platforms built upon such models at the Microsoft Research Forum.<\/p>\n","protected":false},"author":42735,"featured_media":1002792,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-content-parent":999513,"footnotes":""},"research-area":[],"msr-locale":[268875],"msr-post-option":[],"class_list":["post-999732","msr-blog-post","type-msr-blog-post","status-publish","has-post-thumbnail","hentry","msr-locale-en_us"],"msr_assoc_parent":{"id":999513,"type":"story"},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/999732","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-blog-post"}],"author":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/users\/42735"}],"version-history":[{"count":18,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/999732\/revisions"}],"predecessor-version":[{"id":1045092,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/999732\/revisions\/1045092"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media\/1002792"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=999732"}],"wp:term":[{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=999732"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=999732"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=999732"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}