{"id":1011570,"date":"2024-03-08T16:52:19","date_gmt":"2024-03-09T00:52:19","guid":{"rendered":"https:\/\/www.microsoft.com\/en-us\/research\/?post_type=msr-blog-post&#038;p=1011570"},"modified":"2024-03-08T16:52:21","modified_gmt":"2024-03-09T00:52:21","slug":"gigapath-foundation-model-for-digital-pathology","status":"publish","type":"msr-blog-post","link":"https:\/\/www.microsoft.com\/en-us\/research\/articles\/gigapath-foundation-model-for-digital-pathology\/","title":{"rendered":"GigaPath: Foundation Model for Digital Pathology"},"content":{"rendered":"\n<p class=\"has-purple-color has-text-color has-link-color wp-elements-0baf7abac812daafdb534bcf80f22852\"><em>Presented by&nbsp;<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/naotous\/\">Naoto Usuyama<\/a>&nbsp;at&nbsp;<strong>Microsoft Research Forum, March 2024<\/strong><\/em><\/p>\n\n\n\n<div class=\"wp-block-media-text has-vertical-margin-none  has-vertical-padding-none  is-stacked-on-mobile has-white-background-color has-background\" style=\"grid-template-columns:25% auto\" data-bi-an=\"media-text\"><figure class=\"wp-block-media-text__media\"><img loading=\"lazy\" decoding=\"async\" width=\"360\" height=\"360\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Naoto-Usuyama_360x360.jpg\" alt=\"Naoto Usuyama\" class=\"wp-image-1012593 size-full\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Naoto-Usuyama_360x360.jpg 360w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Naoto-Usuyama_360x360-300x300.jpg 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Naoto-Usuyama_360x360-150x150.jpg 150w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Naoto-Usuyama_360x360-180x180.jpg 180w\" sizes=\"(max-width: 360px) 100vw, 360px\" \/><\/figure><div class=\"wp-block-media-text__content\" data-bi-an=\"media-text\">\n<blockquote class=\"wp-block-quote is-style-spectrum is-layout-flow wp-block-quote-is-layout-flow\">\n<p>\u201cThis project (GigaPath) is not possible without many, many collaborators, and we are just scratching the surface, so I\u2019m very excited, and I really hope we can unlock the full potential of the real-world patient data and advance AI for cancer care and research.\u201d<\/p>\n<cite><em>\u2013<\/em> Naoto Usuyama, Principal Researcher, Microsoft Research Health Futures<\/cite><\/blockquote>\n<\/div><\/div>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"GigaPath: Foundation Model for Digital Pathology\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube-nocookie.com\/embed\/eB0SNbTHfl8?feature=oembed&rel=0\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<div class=\"annotations \" data-bi-aN=\"margin-callout\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 annotations__list--right\">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Summarize%20how%20GigaPath%20is%20approaching%20cancer%20pathology.\" target=\"_blank\" aria-label=\"Summarize how GigaPath is approaching cancer pathology.\" data-bi-type=\"annotated-link\" data-bi-cN=\"Summarize how GigaPath is approaching cancer pathology.\" class=\"annotations__list-thumbnail\" >\n\t\t\t\t\t<img loading=\"lazy\" decoding=\"async\" width=\"172\" height=\"96\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png\" class=\"mb-2\" alt=\"Ask Microsoft research copilot experience\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png 240w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-300x169.png 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1024x576.png 1024w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-768x432.png 768w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1066x600.png 1066w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-655x368.png 655w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-343x193.png 343w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-640x360.png 640w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-960x540.png 960w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1280x720.png 1280w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo.png 1400w\" sizes=\"(max-width: 172px) 100vw, 172px\" \/>\t\t\t\t<\/a>\n\t\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Microsoft research copilot experience<\/span>\n\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=Summarize%20how%20GigaPath%20is%20approaching%20cancer%20pathology.\" target=\"_blank\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Summarize how GigaPath is approaching cancer pathology.\" data-bi-aN=\"margin-callout\" data-bi-cN=\"Summarize how GigaPath is approaching cancer pathology.\">\n\t\t\t\tSummarize how GigaPath is approaching cancer pathology.&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n\n\n<div class=\"wp-block-msr-show-more\">\n\t<div class=\"bg-neutral-100 p-5\">\n\t\t<div class=\"show-more-show-less\" data-mount=\"show-more-show-less\">\n\t\t\t<div>\n\t\t\t\t<span>\n\t\t\t\t\t\n\n<h3 class=\"wp-block-heading\" id=\"transcript-lightning-talk-4\">Transcript: Lightning Talk 4<\/h3>\n\n\n\n<p><strong>GigaPath: Foundation model for digital pathology<\/strong><\/p>\n\n\n\n<p><strong>Naoto Usuyama<\/strong>, Principal Researcher, Microsoft Research Health Futures<\/p>\n\n\n\n<p>Naoto Usuyama proposes GigaPath, a novel approach for training large vision transformers for gigapixel pathology images, utilizing a diverse real-world cancer patient dataset, with the goal of laying a foundation for cancer pathology AI.<\/p>\n\n\n\n<p>Microsoft Research Forum, March 5, 2024<\/p>\n\n\n\n\t\t\t\t<\/span>\n\t\t\t\t<span id=\"show-more-show-less-toggle-1\" class=\"show-more-show-less-toggleable-content\">\n\t\t\t\t\t\n\n\n\n<p><strong>NAOTO USUYAMA<\/strong>: Hi, my name is Naoto. I\u2019m from Microsoft Health Futures. I\u2019m excited to talk about GigaPath.<\/p>\n\n\n\n<p>Unfortunately, almost everyone gets cancer at some point. And when cancer is suspected, a small portion is taken from a patient, and this small portion is sent to a pathology lab. The pathology lab prepares a sample and creates a pathology slide. And then the pathology slide is examined under a microscope. And this microscopic view provides lots of information into cancer characteristics, profiles, and this information is essential for choosing the best treatment for each patient.<\/p>\n\n\n\n<p>One notable example is immunotherapy. Immunotherapy is, like, one of the cutting-edge cancer treatments, and it works by using a patient\u2019s own immune system, and it\u2019s, like, a new hope for cancer patients. But unfortunately, it doesn\u2019t work for everyone. The key is the tumor microenvironment. Tumor microenvironment means a complex ecosystem within and around the tumor. This includes not just the cancer cells but also normal cells, like immune cells and blood vessels, and how they interact with each other affects the immunotherapy and the success rate. So modeling the pathology images and modeling the tumor microenvironment is very critical.<\/p>\n\n\n\n<p>My slide is not working. OK, thank you \u2026<\/p>\n\n\n\n<p>The pathology image is super detailed, and the size is huge; one file can be a couple gigabytes. And this pathology slide, I\u2019m not sure if you know, but it\u2019s very tiny, only a few centimeters. But with a microscope, you get very high-resolution images, and it can be 120,000 pixels in just one slide. And this size can blow up transformers easily. Typically, vision transformers use only a few hundred tokens, but for us, we get 56 million, so a few hundred tokens and 56 million. And even if we use a larger patch size, we get a lot of tokens. So it is quite challenging to model pathology slide images. So how do we do this?<\/p>\n\n\n\n<p>We are investigating scalable architectures, and one example is <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/longnet-scaling-transformers-to-1000000000-tokens\/\">LongNet<\/a>. We are collaborating with Microsoft Research Asia, and the key idea is dilated attention. This dilated attention uses sparse attention patterns instead of dense attention in vanilla transformers. And also, we segment the sequence into smaller blocks and then focus attention within this smaller segment. So sparsity and segmentation make it much more scalable. And we are testing this LongNet idea for pathology images, and that\u2019s the modeling side. And data is critical for foundation models, of course, and we are working with Providence Hospital. Providence Hospital is one of the largest nonprofit hospitals in the US, and together, we are working on creating a large-scale, real-world patient dataset. Our dataset includes more than 1 million cancer patient records. This includes all the clinical notes so text, as well, and genomics data, as well, and radiology images and reports, and, of course, the pathology images. So this is very large scale but also multimodal, longitudinal. So this rich dataset enables us to train a large-scale foundation model. And to make the most of the data, we are exploring self-supervised learning approaches in many ways, like unimodal, multimodal, longitudinal, and that\u2019s basically the GigaPath project: to make the real-world foundation model using the Providence Hospital data.<\/p>\n\n\n\n<p>This project is not possible without many, many collaborators, and we are just scratching the surface, so I\u2019m very excited, and I really hope we can unlock the full potential of the real-world patient data and advance AI for cancer care and research.<\/p>\n\n\n\n<p>Thank you.<\/p>\n\n\t\t\t\t<\/span>\n\t\t\t<\/div>\n\t\t\t<button\n\t\t\t\tclass=\"action-trigger glyph-prepend mt-2 mb-0 show-more-show-less-toggle\"\n\t\t\t\taria-expanded=\"false\"\n\t\t\t\tdata-show-less-text=\"Show less\"\n\t\t\t\ttype=\"button\"\n\t\t\t\taria-controls=\"show-more-show-less-toggle-1\"\n\t\t\t\taria-label=\"Show more content\"\n\t\t\t\tdata-alternate-aria-label=\"Show less content\">\n\t\t\t\tShow more\t\t\t<\/button>\n\t\t<\/div>\n\t<\/div>\n<\/div>\n\n\n\n<h2 class=\"wp-block-heading alignwide\" id=\"related-resources\">Related resources<\/h2>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Research Lab<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/lab\/microsoft-health-futures\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Microsoft Health Futures\" data-bi-aN=\"citation\" data-bi-cN=\"Microsoft Health Futures\">\n\t\t\t\tMicrosoft Health Futures&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/longnet-scaling-transformers-to-1000000000-tokens\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"LongNet: Scaling Transformers to 1,000,000,000 Tokens\" data-bi-aN=\"citation\" data-bi-cN=\"LongNet: Scaling Transformers to 1,000,000,000 Tokens\">\n\t\t\t\tLongNet: Scaling Transformers to 1,000,000,000 Tokens&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\"><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>Naoto Usuyama proposes GigaPath, a novel approach for training large vision transformers for gigapixel pathology images, utilizing a diverse real-world cancer patient dataset, with the goal of laying a foundation for cancer pathology AI.<\/p>\n","protected":false},"author":42735,"featured_media":1012590,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-content-parent":1009965,"footnotes":""},"research-area":[],"msr-locale":[268875],"msr-post-option":[],"class_list":["post-1011570","msr-blog-post","type-msr-blog-post","status-publish","has-post-thumbnail","hentry","msr-locale-en_us"],"msr_assoc_parent":{"id":1009965,"type":"story"},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011570"}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-blog-post"}],"author":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/users\/42735"}],"version-history":[{"count":6,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011570\/revisions"}],"predecessor-version":[{"id":1013652,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011570\/revisions\/1013652"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media\/1012590"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=1011570"}],"wp:term":[{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=1011570"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=1011570"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=1011570"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}