1033 lines
281 KiB
HTML
1033 lines
281 KiB
HTML
|
||
|
||
|
||
|
||
|
||
|
||
<!DOCTYPE html>
|
||
<html
|
||
lang="en"
|
||
|
||
data-color-mode="auto" data-light-theme="light" data-dark-theme="dark"
|
||
data-a11y-animated-images="system" data-a11y-link-underlines="true"
|
||
|
||
>
|
||
|
||
|
||
|
||
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<link rel="dns-prefetch" href="https://github.githubassets.com">
|
||
<link rel="dns-prefetch" href="https://avatars.githubusercontent.com">
|
||
<link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
|
||
<link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
|
||
<link rel="preconnect" href="https://github.githubassets.com" crossorigin>
|
||
<link rel="preconnect" href="https://avatars.githubusercontent.com">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="light" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-f4bf1142976e4bbf.css" /><link data-color-theme="light_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind_high_contrast-f661b49995ba0bd8.css" /><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-0b38d22346321c92.css" /><link data-color-theme="light_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia_high_contrast-f1c62c9e70259b9f.css" /><link data-color-theme="dark" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-f50cacf0a86b9929.css" /><link data-color-theme="dark_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind_high_contrast-e61d4f4ca17852c2.css" /><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-39c10993d5603fac.css" /><link data-color-theme="dark_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia_high_contrast-73236c840c0c7d90.css" /><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-0de76f07cc035b10.css" /><link data-color-theme="dark_dimmed_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed_high_contrast-fd1500c8744e40d6.css" />
|
||
|
||
<style type="text/css">
|
||
:root {
|
||
--tab-size-preference: 4;
|
||
}
|
||
|
||
pre, code {
|
||
tab-size: var(--tab-size-preference);
|
||
}
|
||
</style>
|
||
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-ed9ca172356fd545.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-1d2c7f7b52a6068b.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-adcabba7b5c5d221.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-552513ad07a183a1.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repository-11ee8a031c040c1a.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-2d56bdb0166c0238.css" />
|
||
|
||
|
||
|
||
<script type="application/json" id="client-env">{"locale":"en","featureFlags":["actions_enable_background_steps","activity_diff_file_tree","activity_repos_file_tree","activity_repos_overview_header","activity_repos_overview_sidebar","agent_author_search_expansion","agent_author_search_expansion_ui_pulls","alternate_user_config_repo","billing_billable_licenses_cost_center_bucket_fix","billing_budget_expiration","billing_copilot_license_budget_use_license_count","billing_cost_center_list_assigned_resources","billing_discount_threshold_notification","block_user_close_content","cca_show_aic_usage_per_session","ccr_files_changed_model_picker","code_quality_enablement_banner_targeting","code_quality_remove_preview","code_view_raf_sticky_lines","codespaces_prebuild_region_target_update","coding_agent_third_party_model_ui","contentful_primer_code_blocks","copilot_agent_snippy","copilot_api_agentic_issue_marshal_yaml","copilot_automations_pagination","copilot_chat_clear_model_selection_for_default_change","copilot_chat_compact_tables","copilot_chat_header_reorder","copilot_chat_max_upsell","copilot_chat_minimize_contextual","copilot_chat_model_picker_promotions","copilot_chat_models_browser_cache","copilot_chat_new_topic_nudge","copilot_chat_reduce_quota_checks","copilot_chat_vision_dotcom_chat_ga_gate","copilot_chat_vision_preview_gate","copilot_css_textarea_autosize","copilot_custom_copilots","copilot_custom_copilots_feature_preview","copilot_diff_reference_context","copilot_duplicate_thread","copilot_extensions_removal_on_marketplace","copilot_fix_failed_workflows_all_skus","copilot_ftp_hyperspace_upgrade_prompt","copilot_hide_hovercard","copilot_immersive_code_block_transition_wrap","copilot_immersive_embedded_draggable","copilot_immersive_embedded_header_button","copilot_immersive_file_block_transition_open","copilot_immersive_file_preview_keep_mounted","copilot_immersive_suggestion_pills","copilot_immersive_task_hyperlinking","copilot_immersive_task_within_chat_thread","copilot_mc_cli_resume_any_users_task","copilot_mission_control_agent_merge_fix_ci","copilot_mission_control_agent_merge_resolve_conflicts","copilot_mission_control_early_stop","copilot_mission_control_environment_list_icons","copilot_mission_control_managed_sandbox_environments","copilot_mission_control_needs_attention","copilot_mission_control_reasoning_effort","copilot_mission_control_sandbox_client_side_clone","copilot_mission_control_sandbox_remote_bypass","copilot_mission_control_session_filters","copilot_mission_control_task_alive_updates","copilot_mission_control_task_sharing","copilot_org_policy_page_focus_mode","copilot_pr_chat_enhancements","copilot_prominent_upgrade_button","copilot_resource_panel","copilot_share_active_subthread","copilot_spaces_ga","copilot_spaces_individual_policies_ga","copilot_spark_handle_nil_friendly_name","copilot_swe_agent_authorization_status_ui","copilot_swe_agent_automation_resource_scoped_writes","copilot_swe_agent_hide_model_picker_if_only_auto","copilot_swe_agent_issue_comment_trigger","copilot_swe_agent_pr_comment_model_picker","copilot_swe_agent_pull_request_comment_trigger","copilot_swe_agent_pull_request_merged_trigger","copilot_swe_agent_pull_request_opened_trigger","copilot_swe_agent_pull_request_synchronize_trigger","copilot_swe_agent_use_subagents","copilot_task_api_github_rest_style","copilot_task_scoped_alive_channel","copilot_token_based_billing","copilot_unconfigured_is_inherited","copilot_user_can_upgrade_plan_field","copilot_workbench_sunset","copilot_workbench_sunset_redirect","copilot_workbench_ubb","dashboard_indexeddb_caching","dashboard_lists_max_age_filter","dashboard_universe_2025_feedback_dialog","flex_cta_groups_mvp","flex_suite_disable_river_accordion_dither","glc_code_quality_repo_settings_workflow_config","hide_github_models_ui","hyperspace_2025_logged_out_batch_1","hyperspace_2025_logged_out_batch_2","hyperspace_2025_logged_out_batch_3","in_product_messaging_datadog_monitoring","ipm_global_transactional_message_copilot","ipm_global_transactional_message_issues","ipm_global_transactional_message_prs","ipm_global_transactional_message_repos","ipm_global_transactional_message_spaces","issue_fields_multi_select","issue_inline_avatars","issue_pinned_views","issue_pinned_views_optimistic_updates","issue_relative_time_micro","issues_dashboard_sso_structured_errors","issues_expanded_file_types","issues_hide_closed_sub_issues","issues_lazy_load_comment_box_suggestions","issues_react_chrome_container_query_fix","labels_archiving","labels_archiving_info","landing_pages_ninetailed","landing_pages_web_vitals_tracking","lifecycle_label_name_updates","marketing_pages_search_explore_provider","memex_default_issue_create_repository","memex_lazy_hydrate_agent_tasks","memex_live_update_hovercard","memex_mwl_filter_field_delimiter","memex_remove_deprecated_type_issue","merge_queue_restricted_pushers_warning","merge_status_checks_refetch_dedupe","merge_status_header_feedback","oauth_authorize_clickjacking_protection","octocaptcha_origin_optimization","primer_react_css_anchor_positioning","primer_react_merged_forwarded_refs","property_definition_empty_state_suggestions","prs_copilot_app_open_action","prs_css_anchor_positioning","prs_new_conversation_comments_api","prs_omit_files_routes","pull_request_copilot_attribution_header","pull_request_overview_panel_edit_description","pull_request_persister","pull_request_stacks_feedback_dialog","pull_request_stacks_rebase_conflict_instructions","pull_request_virtualization_image_estimate","pull_request_virtualization_scroll_compensation","pull_request_virtualization_scroll_intent","pulls_dashboard_sidebar_layout","quick_search_lazy_suggestions","react_blob_isolate_code_lines","react_blob_ssr_content_visibility","react_data_router_tanstack_allowed","react_query_props_with_key","react_sandbox_future_tanstack","repo_app_actions_workflows","repo_app_agents","repo_app_commits","repo_app_issues","repo_app_pull_requests","repo_app_turbo","repo_issues_sidebar_layout","repo_overview_ask_copilot","repos_contributors_limited_default_range","review_involves_filter","rule_ignored_file_paths","rulesets_actor_list_editor","sample_network_conn_type","security_center_artifact_filters_popover","see_who_reacted","semantic_similarity_duplicate_issue_detection","session_logs_ungroup_reasoning_text","set_sha256_on_repo_creation_form","site_banner_desktop_copilot_app","site_ghca_pixel_mona","site_github_app_business_enterprise_signup_ctas","site_github_app_ga_page","site_github_app_ga_page_highlight","site_github_app_mobile_native_share","site_global_banner_copilot_day","site_global_banner_dev_days_attendee","site_global_banner_learn_copilot_sdk","site_global_nav_spark_models_removed","spark_prompt_secret_scanning","spark_server_connection_status","suppress_automated_browser_vitals","swp_forms_disable_octocaptcha","thread_resolution_reason","ui_service_compute_proxy_control","ui_service_referrer_metrics","update_issue_suggestions","viewscreen_sandbox","warn_inaccessible_attachments","webp_support","workbench_store_readonly"],"githubDomain":"https://github.com","copilotApiOverrideUrl":"https://api.githubcopilot.com","cmcApiUrl":"https://api.github.com/cmc_internal/api"}</script>
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/high-contrast-cookie-e3d808ee18eb9784.js"></script>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wp-runtime-7bcd7c7e71a52839.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-foundation-13e668e8647c754c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-runtime-79f2476139a75354.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fetch-utilities-5305162aa9984914.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ser-cd1b421ad0ad7ed5.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/environment-b2322dc76dbdb954.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/app-runtime.7a9535dabdf3b702.module.css" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/catalyst-044fa3a2acdaf0a9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/selector-observer-e88088f989b27670.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/relative-time-element-85cb37305f54dec4.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/by-6b3c07383371bf58.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ja9-36f036c13c2e3a4c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/kw-866fd9513ae95106.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/r1-b075aab3204f32f9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/jz5-7c5d1669717041f8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hj-5126390ff433704d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j05-dce4d588268f0f7e.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/github-elements-b47746c8206d1338.js" defer="defer"></script>
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/element-registry-abfb59979f5a10d7.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/runtime-helpers-f3f5d71440009c48.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aria-live-76b18916d0c8ec4d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hotkey-5109c77d7ac61078.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-core-7fa41ffc30596003.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-lib-33858e668cc159ef.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/04-2e0b5b7403cb7b99.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ur-024971cb63acad59.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4t-9933bcccf9a9524f.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/6n-71c0ae3e489775ef.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/yhq-3a209035a30b1a09.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/z1-43f07743b557bfbe.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wo-09f85d22a6c8931b.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/9y-75e8f8c6f5eccd49.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j0-e9bfed51669329e8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0p-c8fed99e921442ca.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/behaviors-971e9b5ce578be14.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/react-core.a447335ec9ae046d.module.css" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-menu-c57df7039780d600.js" defer="defer"></script>
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/primer-react-fad6c97f07e0280a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aq-0dbf7d12b4d37e14.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j6-1e8b001d24ed9cac.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/t7l-0a8b07066f584f39.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fe-7576d9b281e2106a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wsr-5fd3071e5a9061ab.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ps6-85a59b16b0ceea7e.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xv-51cb34754bda73a9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/n4u-6317ce1c31aed1db.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j8s-d05eedc69cebc200.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/la-de176c4ee471f2b3.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/u1-c2354415da044847.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/os-1dee58ca1675cc41.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/50-a6c15ac2de24205d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0o-6dedd6c7dfa5a7b9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nsi-89f8558c22a1e954.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3np-b8ac617a3f77670d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/on-75d8015f7c16be87.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bi0-fa59350332fdd7ab.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/b1-e65db5e6cf097025.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0l5-16b0cf9b35fe7f75.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/c2-fb6736e73d800b4e.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/f9-6f78c2d90591da73.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0k-f05429fb3eb4e9a3.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/sp-2bfa60fc59b501fb.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/og-78525ede7a2add85.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/q5-23ee69a34fd8c463.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/qod-dbecf5f56ddaa69a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/du-f925771a20e38027.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ql0-324fc36dae3b9a42.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/20x-dad6ee9ff6177544.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/dsj-342cb45d382b6fdb.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/6f-ddbc6e6b37135f96.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/e84-85a4f8740b573283.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0pz-7d0eea7df68b2d64.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7mh-b8dbc13ace044f4c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bsb-4f4ac8c20da73fac.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/46-2db674c6bc8272ab.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/q1-590dbf4434fdc3ad.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/me-6d77019ac8eb6290.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0d-623095bb226c1f11.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xc-7fd1e6996fa23469.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7c-42dc52617c4fd396.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4b-cfa20397aa7c0823.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/dynamic-github-ui--code-view--route-components-50d91ad640564ff8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/gt-b75626ea0831d606.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-view-37f0af91489560c8.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/ql0.16c145071702f131.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/46.73d41df1aa9a8bca.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/xc.678a3a89a0b97d69.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/4b.b27df37732ef81c7.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dynamic-github-ui--code-view--route-components.c7b328225801cde0.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-view.966b9076e66f352f.module.css" />
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/18-30bd8f55db757032.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/31k-1c94735cfe3b279c.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/notifications-subscriptions-menu-46d0fb840a16af3a.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.27e03c7e4cc2aafc.module.css" />
|
||
|
||
|
||
<title>Postmortem-Report-Reviews/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md at master · Operations-Incident-Board/Postmortem-Report-Reviews · GitHub</title>
|
||
|
||
|
||
|
||
<meta name="route-pattern" content="/:user_id/:repository/blob/*name(/*path)" data-turbo-transient>
|
||
<meta name="route-controller" content="blob" data-turbo-transient>
|
||
<meta name="route-action" content="show" data-turbo-transient>
|
||
<meta name="fetch-nonce" content="v2:8b7429c5-641a-4226-b962-495ecc855454">
|
||
|
||
|
||
<meta name="current-catalog-service-hash" content="f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb">
|
||
|
||
|
||
<meta name="request-id" content="E8B4:197DFA:DF849:171554:6AA369C8" data-pjax-transient="true"/><meta name="html-safe-nonce" content="b6454a9e738e47d56a508645b6f53c2dbbeae296dc39e3188af6f5f857c1bb9f" data-pjax-transient="true"/><meta name="visitor-payload" content="eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJFOEI0OjE5N0RGQTpERjg0OToxNzE1NTQ6NkFBMzY5QzgiLCJ2aXNpdG9yX2lkIjoiMTg3NjYyODcwMTI3MjMwNDA3MiIsInJlZ2lvbl9lZGdlIjoid2VzdHVzMyIsInJlZ2lvbl9yZW5kZXIiOiJ3ZXN0dXMzIn0=" data-pjax-transient="true"/><meta name="visitor-hmac" content="79625d84a8232c0ebe870a7ae2e7c87bb6abfcdcc022efbcb553a77b3a3f55e8" data-pjax-transient="true"/>
|
||
|
||
|
||
<meta name="hovercard-subject-tag" content="repository:52826457" data-turbo-transient>
|
||
|
||
|
||
<meta name="github-keyboard-shortcuts" content="repository,source-code,file-tree,copilot" data-turbo-transient="true" />
|
||
|
||
|
||
<meta name="selected-link" value="repo_source" data-turbo-transient>
|
||
<link rel="assets" href="https://github.githubassets.com/">
|
||
|
||
<meta name="google-site-verification" content="Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I">
|
||
|
||
<meta name="octolytics-url" content="https://collector.github.com/github/collect" />
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="analytics-location" content="/<user-name>/<repo-name>/blob/show" data-turbo-transient="true" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="user-login" content="">
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width">
|
||
|
||
|
||
|
||
<meta name="description" content="Reviews of published Postmortem Reports. Contribute to Operations-Incident-Board/Postmortem-Report-Reviews development by creating an account on GitHub.">
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
|
||
|
||
<link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
|
||
<meta property="fb:app_id" content="1401488693436528">
|
||
<meta name="apple-itunes-app" content="app-id=1477376905, app-argument=https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" />
|
||
|
||
<meta name="twitter:image" content="https://opengraph.githubassets.com/9e73c0c5c143708f0d08cca0821b601cfd07f8eae8f03eb64be72be439b09733/Operations-Incident-Board/Postmortem-Report-Reviews" /><meta name="twitter:site" content="@github" /><meta name="twitter:card" content="summary_large_image" /><meta name="twitter:title" content="Postmortem-Report-Reviews/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md at master · Operations-Incident-Board/Postmortem-Report-Reviews" /><meta name="twitter:description" content="Reviews of published Postmortem Reports. Contribute to Operations-Incident-Board/Postmortem-Report-Reviews development by creating an account on GitHub." />
|
||
<meta property="og:image" content="https://opengraph.githubassets.com/9e73c0c5c143708f0d08cca0821b601cfd07f8eae8f03eb64be72be439b09733/Operations-Incident-Board/Postmortem-Report-Reviews" /><meta property="og:image:alt" content="Reviews of published Postmortem Reports. Contribute to Operations-Incident-Board/Postmortem-Report-Reviews development by creating an account on GitHub." /><meta property="og:image:width" content="1200" /><meta property="og:image:height" content="600" /><meta property="og:site_name" content="GitHub" /><meta property="og:type" content="object" /><meta property="og:title" content="Postmortem-Report-Reviews/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md at master · Operations-Incident-Board/Postmortem-Report-Reviews" /><meta property="og:url" content="https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" /><meta property="og:description" content="Reviews of published Postmortem Reports. Contribute to Operations-Incident-Board/Postmortem-Report-Reviews development by creating an account on GitHub." />
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="hostname" content="github.com">
|
||
|
||
|
||
|
||
<meta name="expected-hostname" content="github.com">
|
||
|
||
|
||
<meta http-equiv="x-pjax-version" content="a4353d0031ff103442a7218793d2d2c957dc34e85dcd18983df1e68836d31b0b" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-csp-version" content="2af6a6627810d190be616096b617c8f37a1419b9435f76190f0e0d7638222ac1" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-css-version" content="b7a4653359baeee3d827ca91d7c149e4962234557341efae58fad4aa5ce82a0a" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-js-version" content="bf86e24ac5dbb66978d7119606653ef175faae865e12d08f190f82a2ba7e04d8" data-turbo-track="reload">
|
||
|
||
<meta name="turbo-cache-control" content="no-preview" data-turbo-transient="">
|
||
|
||
<meta name="turbo-cache-control" content="no-cache" data-turbo-transient>
|
||
|
||
<meta data-hydrostats="publish">
|
||
|
||
<meta name="go-import" content="github.com/Operations-Incident-Board/Postmortem-Report-Reviews git https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews.git">
|
||
|
||
<meta name="octolytics-dimension-user_id" content="17555870" /><meta name="octolytics-dimension-user_login" content="Operations-Incident-Board" /><meta name="octolytics-dimension-repository_id" content="52826457" /><meta name="octolytics-dimension-repository_nwo" content="Operations-Incident-Board/Postmortem-Report-Reviews" /><meta name="octolytics-dimension-repository_public" content="true" /><meta name="octolytics-dimension-repository_is_fork" content="false" /><meta name="octolytics-dimension-repository_network_root_id" content="52826457" /><meta name="octolytics-dimension-repository_network_root_nwo" content="Operations-Incident-Board/Postmortem-Report-Reviews" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="turbo-body-classes" content="logged-out env-production page-responsive">
|
||
<meta name="disable-turbo" content="false">
|
||
|
||
|
||
<meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
|
||
|
||
|
||
<meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
|
||
|
||
<meta name="release" content="b916c7b237679d21f5b2d1609c0234b24d1ee14d" data-turbo-track="reload">
|
||
<meta name="ui-target" content="full">
|
||
|
||
<link rel="mask-icon" href="https://github.githubassets.com/assets/pinned-octocat-093da3e6fa40.svg" color="#000000">
|
||
<link rel="alternate icon" class="js-site-favicon" type="image/png" href="https://github.githubassets.com/favicons/favicon.png">
|
||
<link rel="icon" class="js-site-favicon" type="image/svg+xml" href="https://github.githubassets.com/favicons/favicon.svg" data-base-href="https://github.githubassets.com/favicons/favicon">
|
||
|
||
<meta name="theme-color" content="#1e2327">
|
||
<meta name="color-scheme" content="light dark" />
|
||
|
||
|
||
<link rel="manifest" href="/manifest.json" crossOrigin="use-credentials">
|
||
|
||
</head>
|
||
|
||
<body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
|
||
<div data-turbo-body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
|
||
<div id="__primerPortalRoot__" style="z-index: 1000; position: absolute; width: 100%;" data-turbo-permanent></div>
|
||
|
||
|
||
<div class="position-relative header-wrapper js-header-wrapper ">
|
||
<a href="#start-of-content" data-skip-target-assigned="false" class="px-2 tmp-py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content">Skip to content</a>
|
||
|
||
<span data-view-component="true" class="progress-pjax-loader Progress position-fixed width-full">
|
||
<span style="width: 0%;" data-view-component="true" class="Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis"></span>
|
||
</span>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/obo5-8a5f0070d148d90b.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-791ea523628ac513.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog.7395b07d1f3cfdcd.module.css" />
|
||
|
||
<react-partial
|
||
partial-name="keyboard-shortcuts-dialog"
|
||
data-ssr="false"
|
||
data-attempted-ssr="false"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"docsUrl":"https://docs.github.com/get-started/accessibility/keyboard-shortcuts"}}</script>
|
||
<div data-target="react-partial.reactRoot"></div>
|
||
</react-partial>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nvz-d8ae5b1b70e399a7.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/mz-504eb598d2e57128.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fz-30a6af7c53dfa4f1.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/lht-d07b466cb387fec6.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/1lf-a0b5bbf5742c0fd3.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3d-82c347d20a13fb90.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/tua-07c0f2fd28675950.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/lazy-react-partial-marketing-header-866751d51954477d.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/marketing-header-7937c3a0a4ff17ad.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-brand-css.206bbd4fd232654e.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/lazy-react-partial-marketing-header.784f79e59a54ffa6.module.css" />
|
||
|
||
<react-partial
|
||
partial-name="marketing-header"
|
||
data-ssr="true"
|
||
data-attempted-ssr="true"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"color_mode":"dark","logged_in":false,"marketing_page":false,"home_path":"/","login_path":"/login?return_to=https%3A%2F%2Fgithub.com%2FOperations-Incident-Board%2FPostmortem-Report-Reviews%2Fblob%2Fmaster%2F2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","signup_path":"/signup?ref_cta=Sign+up\u0026ref_loc=header+logged+out\u0026ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow\u0026source=header-repo\u0026source_repo=Operations-Incident-Board%2FPostmortem-Report-Reviews","signup_enabled":true,"is_signup_controller":false,"show_search_and_nav":true,"hide_search":false,"private_mode_enabled":false,"should_use_dotcom_links":true,"auth_hydro_click":"{\"event_type\":\"authentication.click\",\"payload\":{\"location_in_page\":\"site header menu\",\"repository_id\":null,\"auth_type\":\"SIGN_UP\",\"originating_url\":\"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md\",\"user_id\":null}}","auth_hydro_click_hmac":"4b5ff8bb36cd369befed91938391886f55c47b94265deda23002f8c52654a262","overlay":false,"fixed":false}}</script>
|
||
<div data-target="react-partial.reactRoot"><div data-color-mode="dark" data-light-theme="light" data-dark-theme="dark"><header class="MarketingHeader-module__root__Tk7n3 HeaderMktg header-logged-out" role="banner" data-marketing-header="true" data-color-mode="dark" data-light-theme="light" data-dark-theme="dark" data-is-top="true"><h2 class="MarketingHeader-module__visuallyHidden__sqKsl">Navigation Menu</h2><button type="button" class="MarketingHeader-module__backdrop__sw4RU" aria-label="Close navigation menu"></button><div class="MarketingHeader-module__bar__mBSyE"><div class="MarketingHeader-module__topRow__yeury"><div class="MarketingHeader-module__toggleSlot__hDxbh"><button type="button" class="HeaderMenuToggle-module__toggle__i8EiC" aria-label="Toggle navigation" aria-expanded="false"><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span></button></div><a href="/" aria-label="Homepage" class="HeaderLogo-module__logo__UFyHI" data-analytics-event="{"action":"homepage","tag":"link","context":"logo","location":"header","label":"homepage_link_logo_header"}"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github" viewBox="0 0 24 24" width="32" height="32" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path></svg></a><div class="AuthCTAs-module__mobileActions__NNzeV"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/login?return_to=https%3A%2F%2Fgithub.com%2FOperations-Incident-Board%2FPostmortem-Report-Reviews%2Fblob%2Fmaster%2F2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" data-analytics-event="{"action":"sign_in","tag":"link","context":"auth_cta","location":"header","label":"sign_in_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="4b5ff8bb36cd369befed91938391886f55c47b94265deda23002f8c52654a262"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_3dd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_3dd_">Appearance settings</div></div></div><div class="MarketingHeader-module__menu__GIy3y"><div class="MarketingHeader-module__menuWrapper__owstH"><nav class="MarketingNavigation-module__nav__W0KYY" aria-label="Global"><ul class="MarketingNavigation-module__list__tFbMb"><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_nd_">Platform<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5knd_">AI CODE CREATION</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot" data-analytics-event="{"action":"github_copilot","tag":"link","context":"platform","location":"navbar","label":"github_copilot_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>GitHub Copilot</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Write better code with AI</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/ai/github-app" data-analytics-event="{"action":"github_copilot_app","tag":"link","context":"platform","location":"navbar","label":"github_copilot_app_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.766 11.328c-2.063-.25-3.516-1.734-3.516-3.656 0-.781.281-1.625.75-2.188-.203-.515-.172-1.609.063-2.062.625-.078 1.468.25 1.968.703.594-.187 1.219-.281 1.985-.281.765 0 1.39.094 1.953.265.484-.437 1.344-.765 1.969-.687.218.422.25 1.515.046 2.047.5.593.766 1.39.766 2.203 0 1.922-1.453 3.375-3.547 3.64.531.344.89 1.094.89 1.954v1.625c0 .468.391.734.86.547C13.781 14.359 16 11.53 16 8.03 16 3.61 12.406 0 7.984 0 3.563 0 0 3.61 0 8.031a7.88 7.88 0 0 0 5.172 7.422c.422.156.828-.125.828-.547v-1.25c-.219.094-.5.156-.75.156-1.031 0-1.64-.562-2.078-1.609-.172-.422-.36-.672-.719-.719-.187-.015-.25-.093-.25-.187 0-.188.313-.328.625-.328.453 0 .844.281 1.25.86.313.452.64.655 1.031.655s.641-.14 1-.5c.266-.265.47-.5.657-.656"></path></svg>GitHub Copilot app</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Direct agents from issue to merge</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/mcp" data-analytics-event="{"action":"mcp_registry","tag":"link","context":"platform","location":"navbar","label":"mcp_registry_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mcp NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.52 1.12a3.578 3.578 0 0 1 6.078 2.98 3.578 3.578 0 0 1 2.982 6.08l-3.292 3.293a.252.252 0 0 0 0 .354l.843.843a.749.749 0 1 1-1.06 1.06l-.844-.843a1.75 1.75 0 0 1 0-2.474L13.52 9.12a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0L7.731 9.03A.75.75 0 0 1 6.67 7.97l2.85-2.85a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0l-4.799 4.8A.75.75 0 0 1 .72 5.92Z"></path><path d="M7.52 3.12a.749.749 0 1 1 1.06 1.06L5.731 7.03A2.079 2.079 0 0 0 8.67 9.97l2.85-2.85a.749.749 0 1 1 1.06 1.06l-2.849 2.85A3.578 3.578 0 0 1 4.67 5.97Z"></path></svg>MCP Registry</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Integrate external tools</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9knd_">DEVELOPER WORKFLOWS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/actions" data-analytics-event="{"action":"actions","tag":"link","context":"platform","location":"navbar","label":"actions_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-workflow NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path></svg>Actions</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Automate any workflow</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/codespaces" data-analytics-event="{"action":"codespaces","tag":"link","context":"platform","location":"navbar","label":"codespaces_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codespaces NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 11.25c0-.966.784-1.75 1.75-1.75h12.5c.966 0 1.75.784 1.75 1.75v3A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm2-9.5C2 .784 2.784 0 3.75 0h8.5C13.216 0 14 .784 14 1.75v5a1.75 1.75 0 0 1-1.75 1.75h-8.5A1.75 1.75 0 0 1 2 6.75Zm1.75-.25a.25.25 0 0 0-.25.25v5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5a.25.25 0 0 0-.25-.25Zm-2 9.5a.25.25 0 0 0-.25.25v3c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-3a.25.25 0 0 0-.25-.25Z"></path><path d="M7 12.75a.75.75 0 0 1 .75-.75h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path></svg>Codespaces</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Instant dev environments</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/issues" data-analytics-event="{"action":"issues","tag":"link","context":"platform","location":"navbar","label":"issues_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-issue-opened NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path></svg>Issues</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Plan and track work</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-review" data-analytics-event="{"action":"code_review","tag":"link","context":"platform","location":"navbar","label":"code_review_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path></svg>Code Review</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Manage code changes</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-quality" data-analytics-event="{"action":"code_quality","tag":"link","context":"platform","location":"navbar","label":"code_quality_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codescan-checkmark NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.28 6.28a.75.75 0 1 0-1.06-1.06L6.25 8.19l-.97-.97a.75.75 0 0 0-1.06 1.06l1.5 1.5a.75.75 0 0 0 1.06 0l3.5-3.5Z"></path><path d="M7.5 15a7.5 7.5 0 1 1 5.807-2.754l2.473 2.474a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2.474-2.473A7.472 7.472 0 0 1 7.5 15Zm0-13.5a6 6 0 1 0 4.094 10.386.748.748 0 0 1 .293-.292 6.002 6.002 0 0 0 1.117-6.486A6.002 6.002 0 0 0 7.5 1.5Z"></path></svg>Code Quality</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enforce quality at merge</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dknd_">APPLICATION SECURITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{"action":"github_advanced_security","tag":"link","context":"platform","location":"navbar","label":"github_advanced_security_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Find and fix vulnerabilities</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/code-security" data-analytics-event="{"action":"code_security","tag":"link","context":"platform","location":"navbar","label":"code_security_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code-square NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>Code security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Secure your code as you build</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/secret-protection" data-analytics-event="{"action":"secret_protection","tag":"link","context":"platform","location":"navbar","label":"secret_protection_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-lock NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M4 4a4 4 0 0 1 8 0v2h.25c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 12.25 15h-8.5A1.75 1.75 0 0 1 2 13.25v-5.5C2 6.784 2.784 6 3.75 6H4Zm8.25 3.5h-8.5a.25.25 0 0 0-.25.25v5.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25ZM10.5 6V4a2.5 2.5 0 1 0-5 0v2Z"></path></svg>Secret protection</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Stop leaks before they start</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ NavGroup-module__hasSeparator__FnMrN"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_hknd_">EXPLORE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_hknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/why-github" data-analytics-event="{"action":"why_github","tag":"link","context":"platform","location":"navbar","label":"why_github_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Why GitHub</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{"action":"documentation","tag":"link","context":"platform","location":"navbar","label":"documentation_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog" data-analytics-event="{"action":"blog","tag":"link","context":"platform","location":"navbar","label":"blog_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Blog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog/changelog" data-analytics-event="{"action":"changelog","tag":"link","context":"platform","location":"navbar","label":"changelog_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Changelog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/marketplace" data-analytics-event="{"action":"marketplace","tag":"link","context":"platform","location":"navbar","label":"marketplace_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Marketplace</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/features" data-analytics-event="{"action":"view_all_features","tag":"link","context":"platform","location":"navbar","label":"view_all_features_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all features</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_17d_">Solutions<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_17d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5l7d_">BY COMPANY SIZE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise" data-analytics-event="{"action":"enterprises","tag":"link","context":"solutions","location":"navbar","label":"enterprises_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Enterprises</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/team" data-analytics-event="{"action":"small_and_medium_teams","tag":"link","context":"solutions","location":"navbar","label":"small_and_medium_teams_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Small and medium teams</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise/startups" data-analytics-event="{"action":"startups","tag":"link","context":"solutions","location":"navbar","label":"startups_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Startups</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/nonprofits" data-analytics-event="{"action":"nonprofits","tag":"link","context":"solutions","location":"navbar","label":"nonprofits_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Nonprofits</span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9l7d_">BY USE CASE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/app-modernization" data-analytics-event="{"action":"app_modernization","tag":"link","context":"solutions","location":"navbar","label":"app_modernization_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">App Modernization</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devsecops" data-analytics-event="{"action":"devsecops","tag":"link","context":"solutions","location":"navbar","label":"devsecops_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevSecOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devops" data-analytics-event="{"action":"devops","tag":"link","context":"solutions","location":"navbar","label":"devops_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/ci-cd" data-analytics-event="{"action":"ci/cd","tag":"link","context":"solutions","location":"navbar","label":"ci/cd_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">CI/CD</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/use-case" data-analytics-event="{"action":"view_all_use_cases","tag":"link","context":"solutions","location":"navbar","label":"view_all_use_cases_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all use cases</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dl7d_">BY INDUSTRY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dl7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/healthcare" data-analytics-event="{"action":"healthcare","tag":"link","context":"solutions","location":"navbar","label":"healthcare_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Healthcare</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/financial-services" data-analytics-event="{"action":"financial_services","tag":"link","context":"solutions","location":"navbar","label":"financial_services_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Financial services</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/manufacturing" data-analytics-event="{"action":"manufacturing","tag":"link","context":"solutions","location":"navbar","label":"manufacturing_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Manufacturing</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/government" data-analytics-event="{"action":"government","tag":"link","context":"solutions","location":"navbar","label":"government_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Government</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/industry" data-analytics-event="{"action":"view_all_industries","tag":"link","context":"solutions","location":"navbar","label":"view_all_industries_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all industries</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions" data-analytics-event="{"action":"view_all_solutions","tag":"link","context":"solutions","location":"navbar","label":"view_all_solutions_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all solutions</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_1nd_">Resources<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_1nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5lnd_">EXPLORE BY TOPIC</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=ai" data-analytics-event="{"action":"ai","tag":"link","context":"resources","location":"navbar","label":"ai_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">AI</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=software-development" data-analytics-event="{"action":"software_development","tag":"link","context":"resources","location":"navbar","label":"software_development_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Software Development</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=devops" data-analytics-event="{"action":"devops","tag":"link","context":"resources","location":"navbar","label":"devops_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=security" data-analytics-event="{"action":"security","tag":"link","context":"resources","location":"navbar","label":"security_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources/articles" data-analytics-event="{"action":"view_all_topics","tag":"link","context":"resources","location":"navbar","label":"view_all_topics_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all topics</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9lnd_">EXPLORE BY TYPE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/customer-stories" data-analytics-event="{"action":"customer_stories","tag":"link","context":"resources","location":"navbar","label":"customer_stories_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer stories</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/events" data-analytics-event="{"action":"events__webinars","tag":"link","context":"resources","location":"navbar","label":"events__webinars_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Events & webinars</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/whitepapers" data-analytics-event="{"action":"ebooks__reports","tag":"link","context":"resources","location":"navbar","label":"ebooks__reports_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Ebooks & reports</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/executive-insights" data-analytics-event="{"action":"business_insights","tag":"link","context":"resources","location":"navbar","label":"business_insights_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Business insights</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://skills.github.com" data-analytics-event="{"action":"github_skills","tag":"link","context":"resources","location":"navbar","label":"github_skills_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Skills</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dlnd_">SUPPORT & SERVICES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dlnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{"action":"documentation","tag":"link","context":"resources","location":"navbar","label":"documentation_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://support.github.com" data-analytics-event="{"action":"customer_support","tag":"link","context":"resources","location":"navbar","label":"customer_support_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer support</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/orgs/community/discussions" data-analytics-event="{"action":"community_forum","tag":"link","context":"resources","location":"navbar","label":"community_forum_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Community forum</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trust-center" data-analytics-event="{"action":"trust_center","tag":"link","context":"resources","location":"navbar","label":"trust_center_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trust center</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/partners" data-analytics-event="{"action":"partners","tag":"link","context":"resources","location":"navbar","label":"partners_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Partners</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources" data-analytics-event="{"action":"view_all_resources","tag":"link","context":"resources","location":"navbar","label":"view_all_resources_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all resources</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_27d_">Open Source<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_27d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5m7d_">COMMUNITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/open-source/sponsors" data-analytics-event="{"action":"github_sponsors","tag":"link","context":"open_source","location":"navbar","label":"github_sponsors_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sponsor-tiers NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.586 1C12.268 1 13.5 2.37 13.5 4.25c0 1.745-.996 3.359-2.622 4.831-.166.15-.336.297-.509.438l1.116 5.584a.75.75 0 0 1-.991.852l-2.409-.876a.25.25 0 0 0-.17 0l-2.409.876a.75.75 0 0 1-.991-.852L5.63 9.519a13.78 13.78 0 0 1-.51-.438C3.497 7.609 2.5 5.995 2.5 4.25 2.5 2.37 3.732 1 5.414 1c.963 0 1.843.403 2.474 1.073L8 2.198l.112-.125a3.385 3.385 0 0 1 2.283-1.068L10.586 1Zm-3.621 9.495-.718 3.594 1.155-.42a1.75 1.75 0 0 1 1.028-.051l.168.051 1.154.42-.718-3.592c-.199.13-.37.235-.505.314l-.169.097a.75.75 0 0 1-.72 0 9.54 9.54 0 0 1-.515-.308l-.16-.105ZM10.586 2.5c-.863 0-1.611.58-1.866 1.459-.209.721-1.231.721-1.44 0C7.025 3.08 6.277 2.5 5.414 2.5 4.598 2.5 4 3.165 4 4.25c0 1.23.786 2.504 2.128 3.719.49.443 1.018.846 1.546 1.198l.325.21.076-.047.251-.163a13.341 13.341 0 0 0 1.546-1.198C11.214 6.754 12 5.479 12 4.25c0-1.085-.598-1.75-1.414-1.75Z"></path></svg>GitHub Sponsors</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Fund open source developers</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9m7d_">PROGRAMS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://securitylab.github.com" data-analytics-event="{"action":"security_lab","tag":"link","context":"open_source","location":"navbar","label":"security_lab_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security Lab</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://maintainers.github.com" data-analytics-event="{"action":"maintainer_community","tag":"link","context":"open_source","location":"navbar","label":"maintainer_community_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Maintainer Community</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://stars.github.com" data-analytics-event="{"action":"github_stars","tag":"link","context":"open_source","location":"navbar","label":"github_stars_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Stars</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://archiveprogram.github.com" data-analytics-event="{"action":"archive_program","tag":"link","context":"open_source","location":"navbar","label":"archive_program_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Archive Program</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dm7d_">REPOSITORIES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dm7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/topics" data-analytics-event="{"action":"topics","tag":"link","context":"open_source","location":"navbar","label":"topics_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Topics</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trending" data-analytics-event="{"action":"trending","tag":"link","context":"open_source","location":"navbar","label":"trending_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trending</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/collections" data-analytics-event="{"action":"collections","tag":"link","context":"open_source","location":"navbar","label":"collections_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Collections</span></a></li></ul></div></li></ul></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_2nd_">Enterprise<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_2nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5mnd_">ENTERPRISE SOLUTIONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise" data-analytics-event="{"action":"enterprise_platform","tag":"link","context":"enterprise","location":"navbar","label":"enterprise_platform_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-stack NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.122.392a1.75 1.75 0 0 1 1.756 0l5.003 2.902c.83.481.83 1.68 0 2.162L8.878 8.358a1.75 1.75 0 0 1-1.756 0L2.119 5.456a1.251 1.251 0 0 1 0-2.162ZM8.125 1.69a.248.248 0 0 0-.25 0l-4.63 2.685 4.63 2.685a.248.248 0 0 0 .25 0l4.63-2.685ZM1.601 7.789a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0L1.874 8.814A.75.75 0 0 1 1.6 7.789Zm0 3.5a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0l-5.248-3.044a.75.75 0 0 1-.273-1.025Z"></path></svg>Enterprise platform</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">AI-powered developer platform</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9mnd_">AVAILABLE ADD-ONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{"action":"github_advanced_security","tag":"link","context":"enterprise","location":"navbar","label":"github_advanced_security_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade security features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot/copilot-business" data-analytics-event="{"action":"copilot_for_business","tag":"link","context":"enterprise","location":"navbar","label":"copilot_for_business_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>Copilot for Business</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade AI features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise/premium-support" data-analytics-event="{"action":"premium_support","tag":"link","context":"enterprise","location":"navbar","label":"premium_support_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-comment-discussion NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1h8.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 10.25 10H7.061l-2.574 2.573A1.458 1.458 0 0 1 2 11.543V10h-.25A1.75 1.75 0 0 1 0 8.25v-5.5C0 1.784.784 1 1.75 1ZM1.5 2.75v5.5c0 .138.112.25.25.25h1a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h3.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13 2a.25.25 0 0 0-.25-.25h-.5a.75.75 0 0 1 0-1.5h.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 14.25 12H14v1.543a1.458 1.458 0 0 1-2.487 1.03L9.22 12.28a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l2.22 2.22v-2.19a.75.75 0 0 1 .75-.75h1a.25.25 0 0 0 .25-.25Z"></path></svg>Premium Support</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade 24/7 support</span></span></a></li></ul></div></li></ul></div></div></li><li><a href="https://github.com/pricing" data-analytics-event="{"action":"pricing","tag":"link","context":"pricing","location":"navbar","label":"pricing_link_pricing_navbar"}" class="MarketingNavigation-module__navLink__hUomM">Pricing</a></li></ul></nav><div class="MarketingHeader-module__ctaContainer__tBmPz"><div class="HeaderSearch-module__searchSlot__oVOUS"><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderSearch-module__trigger__zsF9q" type="button" aria-haspopup="dialog" aria-expanded="false" aria-label="Search or jump to, type / to search" data-analytics-event="{"action":"searchbar","tag":"input","context":"global","location":"navbar","label":"searchbar_input_global_navbar"}"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"><span class="HeaderSearch-module__content__kMpxU"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search HeaderSearch-module__icon__wcrHX" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg><span class="HeaderSearch-module__label__d1iWG">Search</span><kbd class="HeaderSearch-module__kbd__HNG0o" aria-hidden="true">/</kbd></span></span></span></button><div class="d-none"></div></div><div class="AuthCTAs-module__signInWrap__q2P60"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq AuthCTAs-module__desktopActionGap__UZuXT AuthCTAs-module__hiddenBelowLg__BfKBw" href="/login?return_to=https%3A%2F%2Fgithub.com%2FOperations-Incident-Board%2FPostmortem-Report-Reviews%2Fblob%2Fmaster%2F2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" data-analytics-event="{"action":"sign_in","tag":"link","context":"auth_cta","location":"header","label":"sign_in_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="4b5ff8bb36cd369befed91938391886f55c47b94265deda23002f8c52654a262"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a></div><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--secondary___gHnw_ Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow&source=header-repo&source_repo=Operations-Incident-Board%2FPostmortem-Report-Reviews" data-analytics-event="{"action":"sign_up","tag":"link","context":"auth_cta","location":"header","label":"sign_up_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="4b5ff8bb36cd369befed91938391886f55c47b94265deda23002f8c52654a262"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-secondary___eJ0_a">Sign up</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_fbd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_fbd_">Appearance settings</div></div></div></div></div><div class="MarketingHeader-module__bottomBorder__uZT38" aria-hidden="true"></div></header></div></div>
|
||
</react-partial>
|
||
|
||
|
||
|
||
<div hidden="hidden" data-view-component="true" class="js-stale-session-flash stale-session-flash flash flash-warn flash-full">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
|
||
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<span class="js-stale-session-flash-signed-in" hidden>You signed in with another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
<span class="js-stale-session-flash-signed-out" hidden>You signed out in another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
<span class="js-stale-session-flash-switched" hidden>You switched accounts on another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
|
||
<button id="icon-button-bd5a3e69-65d0-437f-aeda-0b58655d0d64" aria-labelledby="tooltip-3bc42abf-1c8f-4904-b8a4-6fa8bdc19f68" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium flash-close js-flash-close"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x Button-visual">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button><tool-tip id="tooltip-3bc42abf-1c8f-4904-b8a4-6fa8bdc19f68" for="icon-button-bd5a3e69-65d0-437f-aeda-0b58655d0d64" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Dismiss alert</tool-tip>
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="start-of-content" class="show-on-focus"></div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="js-flash-container" class="flash-container" data-turbo-replace>
|
||
|
||
|
||
|
||
|
||
|
||
<template class="js-flash-template">
|
||
|
||
<div class="flash flash-full {{ className }}">
|
||
<div >
|
||
<button autofocus class="flash-close js-flash-close" type="button" aria-label="Dismiss this message">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
<div aria-atomic="true" role="alert" class="js-flash-alert">
|
||
|
||
<div>{{ message }}</div>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</template>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div
|
||
class="application-main "
|
||
data-commit-hovercards-enabled
|
||
data-discussion-hovercards-enabled
|
||
data-issue-and-pr-hovercards-enabled
|
||
data-project-hovercards-enabled
|
||
>
|
||
<div itemscope itemtype="http://schema.org/SoftwareSourceCode" class="">
|
||
<main id="js-repo-pjax-container" >
|
||
|
||
|
||
|
||
|
||
<include-fragment src="/in-product-messaging/budget-threshold-banner?organization=Operations-Incident-Board" data-nonce="v2:8b7429c5-641a-4226-b962-495ecc855454" data-view-component="true">
|
||
|
||
<div data-show-on-forbidden-error hidden>
|
||
<div class="Box">
|
||
<div class="blankslate-container">
|
||
<div data-view-component="true" class="blankslate blankslate-spacious color-bg-default rounded-2">
|
||
|
||
|
||
<h3 data-view-component="true" class="blankslate-heading"> Uh oh!
|
||
</h3>
|
||
<p data-view-component="true" class="blankslate-description"> <p class="color-fg-muted my-2 mb-2 ws-normal">There was an error while loading. <a class="Link--inTextBlock" data-turbo="false" href="" aria-label="Please reload this page">Please reload this page</a>.</p>
|
||
</p>
|
||
|
||
</div> </div>
|
||
</div> </div>
|
||
</include-fragment>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="repository-container-header" class="tmp-pt-3 hide-full-screen" style="background-color: var(--page-header-bgColor, var(--color-page-header-bg));" data-turbo-replace>
|
||
|
||
<div class="d-flex flex-nowrap flex-justify-end tmp-mb-3 tmp-px-3 tmp-px-lg-5" style="gap: 1rem;">
|
||
|
||
<div class="flex-auto min-width-0 width-fit">
|
||
|
||
<div class=" d-flex flex-wrap flex-items-center wb-break-word f3 text-normal">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo color-fg-muted mr-2 tmp-mr-2">
|
||
<path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path>
|
||
</svg>
|
||
|
||
<span class="author flex-self-stretch" itemprop="author">
|
||
<a class="url fn" rel="author" data-hovercard-type="organization" data-hovercard-url="/orgs/Operations-Incident-Board/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="/Operations-Incident-Board">
|
||
Operations-Incident-Board
|
||
</a> </span>
|
||
<span class="mx-1 flex-self-stretch color-fg-muted">/</span>
|
||
<strong itemprop="name" class="mr-2 flex-self-stretch">
|
||
<a data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" href="/Operations-Incident-Board/Postmortem-Report-Reviews">Postmortem-Report-Reviews</a>
|
||
</strong>
|
||
|
||
<span></span><span class="Label Label--secondary v-align-middle mr-1">Public</span>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div id="repository-details-container" class="flex-shrink-0" data-turbo-replace style="max-width: 70%;">
|
||
<ul class="pagehead-actions flex-shrink-0 d-none d-md-inline" style="padding: 2px 0;">
|
||
|
||
|
||
|
||
<li>
|
||
<a href="/login?return_to=%2FOperations-Incident-Board%2FPostmortem-Report-Reviews" rel="nofollow" id="repository-details-watch-button" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"notification subscription menu watch","repository_id":null,"auth_type":"LOG_IN","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="453826bc8634a00d856ac79a778e75c170e327b257d3358cdbb46e4e5cda6ead" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2 tmp-mr-2">
|
||
<path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path>
|
||
</svg>Notifications
|
||
</a> <tool-tip id="tooltip-7c7c9d16-b3ab-4e53-8d78-75950d073ee8" for="repository-details-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute">You must be signed in to change notification settings</tool-tip>
|
||
|
||
</li>
|
||
|
||
<li>
|
||
<a icon="repo-forked" id="fork-button" href="/login?return_to=%2FOperations-Incident-Board%2FPostmortem-Report-Reviews" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"repo details fork button","repository_id":52826457,"auth_type":"LOG_IN","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="80e7441a5c2a4bd1f408821ae4972adffecbc2a1e0b36c92589776e6bfc7d02c" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2 tmp-mr-2">
|
||
<path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
|
||
</svg>Fork
|
||
<span id="repo-network-counter" data-pjax-replace="true" data-turbo-replace="true" title="9" data-view-component="true" class="Counter">9</span>
|
||
</a>
|
||
</li>
|
||
|
||
<li>
|
||
<div data-view-component="true" class="BtnGroup d-flex">
|
||
<a href="/login?return_to=%2FOperations-Incident-Board%2FPostmortem-Report-Reviews" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"star button","repository_id":52826457,"auth_type":"LOG_IN","originating_url":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","user_id":null}}" data-hydro-click-hmac="cf8140421bc9048b4a58bfa56d96fe8cfa2cc30012744701e8947de2d00c3423" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2 tmp-mr-2">
|
||
<path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
|
||
</svg><span data-view-component="true" class="d-inline">
|
||
Star
|
||
</span> <span id="repo-stars-counter-star" aria-label="57 users starred this repository" data-singular-suffix="user starred this repository" data-plural-suffix="users starred this repository" data-turbo-replace="true" title="57" data-view-component="true" class="Counter js-social-count">57</span>
|
||
</a></div>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="responsive-meta-container" data-turbo-replace>
|
||
</div>
|
||
|
||
|
||
<nav data-pjax="#js-repo-pjax-container" aria-label="Repository" data-view-component="true" class="js-repo-nav js-sidenav-container-pjax js-responsive-underlinenav overflow-hidden UnderlineNav px-3 tmp-px-3 px-md-4 tmp-px-md-4 px-lg-5 tmp-px-lg-5">
|
||
|
||
<ul data-view-component="true" class="UnderlineNav-body list-style-none">
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="code-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews" data-tab-item="i0code-tab" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches repo_packages repo_deployments repo_attestations /Operations-Incident-Board/Postmortem-Report-Reviews" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g c" data-command-id="repositories:go-to-code" data-react-nav="code-view" data-react-nav-anchor="code-view-repo-link" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Code","target":"UNDERLINE_NAV.TAB"}" aria-current="page" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item selected">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
<span data-content="Code">Code</span>
|
||
<span id="code-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="issues-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/issues" data-tab-item="i1issues-tab" data-selected-links="repo_issues repo_labels repo_milestones /Operations-Incident-Board/Postmortem-Report-Reviews/issues" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g i" data-command-id="repositories:go-to-issues" data-react-nav="issues-react" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Issues","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
|
||
</svg>
|
||
<span data-content="Issues">Issues</span>
|
||
<span id="issues-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="pull-requests-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/pulls" data-tab-item="i2pull-requests-tab" data-selected-links="repo_pulls checks /Operations-Incident-Board/Postmortem-Report-Reviews/pulls" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g p" data-command-id="repositories:go-to-pull-requests" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Pull requests","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
|
||
</svg>
|
||
<span data-content="Pull requests">Pull requests</span>
|
||
<span id="pull-requests-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="actions-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/actions" data-tab-item="i3actions-tab" data-selected-links="repo_actions /Operations-Incident-Board/Postmortem-Report-Reviews/actions" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g a" data-command-id="repositories:go-to-actions" data-react-nav="actions-workflows" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Actions","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
|
||
</svg>
|
||
<span data-content="Actions">Actions</span>
|
||
<span id="actions-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="projects-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/projects" data-tab-item="i4projects-tab" data-selected-links="repo_projects new_repo_project repo_project /Operations-Incident-Board/Postmortem-Report-Reviews/projects" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g b" data-command-id="repositories:go-to-projects" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Projects","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
|
||
</svg>
|
||
<span data-content="Projects">Projects</span>
|
||
<span id="projects-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="security-and-quality-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/security" data-tab-item="i5security-and-quality-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /Operations-Incident-Board/Postmortem-Report-Reviews/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-command-id="repositories:go-to-security" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Security and quality","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<span data-content="Security and quality">Security and quality</span>
|
||
<span id="security-and-quality-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="insights-tab" href="/Operations-Incident-Board/Postmortem-Report-Reviews/pulse" data-tab-item="i6insights-tab" data-selected-links="repo_graphs repo_contributors dependency_graph dependabot_updates pulse people community /Operations-Incident-Board/Postmortem-Report-Reviews/pulse" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-command-id="repositories:go-to-insights" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Insights","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
|
||
</svg>
|
||
<span data-content="Insights">Insights</span>
|
||
<span id="insights-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
</ul>
|
||
<div style="visibility:hidden;" data-view-component="true" class="UnderlineNav-actions js-responsive-underlinenav-overflow position-absolute pr-3 tmp-pr-3 pr-md-4 tmp-pr-md-4 pr-lg-5 tmp-pr-lg-5 right-0"> <action-menu data-select-variant="none" data-view-component="true">
|
||
<focus-group direction="vertical" mnemonics retain>
|
||
<button id="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-button" popovertarget="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-overlay" aria-controls="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-list" aria-haspopup="true" aria-labelledby="tooltip-4f0aa29d-db00-4875-8387-09e6cbd1c881" type="button" data-view-component="true" class="Button Button--iconOnly Button--secondary Button--medium UnderlineNav-item"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-kebab-horizontal Button-visual">
|
||
<path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path>
|
||
</svg>
|
||
</button><tool-tip id="tooltip-4f0aa29d-db00-4875-8387-09e6cbd1c881" for="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-button" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Additional navigation options</tool-tip>
|
||
|
||
|
||
<anchored-position data-target="action-menu.overlay" id="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-overlay" anchor="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-button" align="start" side="outside-bottom" anchor-offset="normal" popover="auto" data-view-component="true">
|
||
<div data-view-component="true" class="Overlay Overlay--size-auto">
|
||
|
||
<div data-view-component="true" class="Overlay-body Overlay-body--paddingNone"> <action-list>
|
||
<div data-view-component="true">
|
||
<ul aria-labelledby="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-button" id="action-menu-73211f33-34ab-4b9a-80da-0ad2220f4c3a-list" role="menu" data-view-component="true" class="ActionListWrap--inset ActionListWrap">
|
||
<li hidden="hidden" data-menu-item="i0code-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-822f9f46-579f-4247-a504-02c5d2a4192c" href="/Operations-Incident-Board/Postmortem-Report-Reviews" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code">
|
||
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Code
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i1issues-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-8b483ca9-0bd8-4e8b-9e11-5068e4de3dbd" href="/Operations-Incident-Board/Postmortem-Report-Reviews/issues" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened">
|
||
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Issues
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i2pull-requests-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-754ec5e8-00d2-4478-8824-23469b543419" href="/Operations-Incident-Board/Postmortem-Report-Reviews/pulls" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request">
|
||
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Pull requests
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i3actions-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-7a4314f7-7b3e-4cee-a519-c9742642fae6" href="/Operations-Incident-Board/Postmortem-Report-Reviews/actions" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play">
|
||
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Actions
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i4projects-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-7a2d979b-c0a2-427a-b904-771ff37562a6" href="/Operations-Incident-Board/Postmortem-Report-Reviews/projects" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table">
|
||
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Projects
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i5security-and-quality-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-b24c2001-070a-429a-bf71-5308b476ba3b" href="/Operations-Incident-Board/Postmortem-Report-Reviews/security" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield">
|
||
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Security and quality
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i6insights-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-1229abd1-d875-4cb9-bd5a-6bb0615a9bf4" href="/Operations-Incident-Board/Postmortem-Report-Reviews/pulse" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph">
|
||
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Insights
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
</ul>
|
||
</div></action-list>
|
||
|
||
|
||
</div>
|
||
|
||
</div></anchored-position> </focus-group>
|
||
</action-menu></div>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<turbo-frame id="repo-content-turbo-frame" target="_top" data-turbo-action="advance" class="">
|
||
<div id="repo-content-pjax-container" class="repository-content " >
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<react-app
|
||
app-name="code-view"
|
||
initial-path="/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md"
|
||
style="display: block; min-height: calc(100vh - 64px);"
|
||
data-attempted-ssr="true"
|
||
data-ssr="true"
|
||
data-lazy="false"
|
||
data-alternate="false"
|
||
data-data-router-enabled="true"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-app.embeddedData">{"payload":{"codeViewBlobRoute":{"csv":null,"csvError":null,"headerInfo":{"toc":[{"level":1,"text":"Report: Google Compute Engine Incident #16007: Connectivity issues in all regions","anchor":"report-google-compute-engine-incident-16007-connectivity-issues-in-all-regions","htmlText":"Report: Google Compute Engine Incident #16007: Connectivity issues in all regions"},{"level":2,"text":"Company Overview","anchor":"company-overview","htmlText":"Company Overview"},{"level":2,"text":"Incident Overview","anchor":"incident-overview","htmlText":"Incident Overview"},{"level":2,"text":"Incident Updates / Postmortem","anchor":"incident-updates--postmortem","htmlText":"Incident Updates / Postmortem"},{"level":3,"text":"Updates During the Incident","anchor":"updates-during-the-incident","htmlText":"Updates During the Incident"},{"level":3,"text":"The postmortem","anchor":"the-postmortem","htmlText":"The postmortem"},{"level":3,"text":"Timeline","anchor":"timeline","htmlText":"Timeline"},{"level":2,"text":"Remediation / Future Work","anchor":"remediation--future-work","htmlText":"Remediation / Future Work"},{"level":2,"text":"Additional thoughts","anchor":"additional-thoughts","htmlText":"Additional thoughts"},{"level":2,"text":"About the Author","anchor":"about-the-author","htmlText":"About the Author"},{"level":6,"text":"#outage #network #bug #technical","anchor":"outage-network-bug-technical","htmlText":"#outage #network #bug #technical"}]},"issueTemplate":null,"discussionTemplate":null,"richText":"\u003carticle class=\"markdown-body entry-content container-lg\" itemprop=\"text\"\u003e\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReport: Google Compute Engine Incident #16007: Connectivity issues in all regions\u003c/h1\u003e\u003ca id=\"user-content-report-google-compute-engine-incident-16007-connectivity-issues-in-all-regions\" class=\"anchor\" aria-label=\"Permalink: Report: Google Compute Engine Incident #16007: Connectivity issues in all regions\" href=\"#report-google-compute-engine-incident-16007-connectivity-issues-in-all-regions\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eLink to Postmortem: \u003ca href=\"https://status.cloud.google.com/incident/compute/16007?post-mortem\" rel=\"nofollow\"\u003ehttps://status.cloud.google.com/incident/compute/16007?post-mortem\u003c/a\u003e\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCompany Overview\u003c/h2\u003e\u003ca id=\"user-content-company-overview\" class=\"anchor\" aria-label=\"Permalink: Company Overview\" href=\"#company-overview\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eGoogle is Google. You may have heard of them. Google Compute Engine is an IaaS offering virtual machines with per-minute billing.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eIncident Overview\u003c/h2\u003e\u003ca id=\"user-content-incident-overview\" class=\"anchor\" aria-label=\"Permalink: Incident Overview\" href=\"#incident-overview\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eGoogle Compute Engine instances suffered increasingly higher latency in internet traffic, ultimately culminating in an 18-minute total internet connectivity outage. Edge routers in Google's network began to stop sending BGP announcements for all of GCE's IP blocks. As the incorrect configuration spread to more edge routers, traffic began taking increasingly less-optimal routes until finally no GCE IP blocks were announced by any edge router.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis incident is especially interesting because it was caused by a combination of three bugs in Google's networking management software, at least one of which was previously unknown. The first bug caused a valid configuration to be rejected, and the second bug caused the system to respond to that rejection by removing \u003cem\u003eall\u003c/em\u003e GCE IP blocks from the configuration. The third bug caused the canary of this erroneous configuration to be accepted despite the failure. Subsequently, the system slowly deployed the erroneous configuration to edge routers, causing the progressive impact.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eIncident Updates / Postmortem\u003c/h2\u003e\u003ca id=\"user-content-incident-updates--postmortem\" class=\"anchor\" aria-label=\"Permalink: Incident Updates / Postmortem\" href=\"#incident-updates--postmortem\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eUpdates During the Incident\u003c/h3\u003e\u003ca id=\"user-content-updates-during-the-incident\" class=\"anchor\" aria-label=\"Permalink: Updates During the Incident\" href=\"#updates-during-the-incident\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eGoogle initially posted that they were investigating an issue with Cloud VPN in asia-east1. Subsequently, Google reported \"severe network connectivity issues in all regions\", followed fairly quickly by an update indicating resolution of the issue.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe postmortem\u003c/h3\u003e\u003ca id=\"user-content-the-postmortem\" class=\"anchor\" aria-label=\"Permalink: The postmortem\" href=\"#the-postmortem\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAbout 36 hours after the incident, Google released a lengthy postmortem. It explains BGP at a high level and details the confluence of bugs that resulted in the outage. Google also apologizes multiple times and offers a blanket service credit of 10% off GCE (vm) charges and 25% off VPN charges for the month.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTimeline\u003c/h3\u003e\u003ca id=\"user-content-timeline\" class=\"anchor\" aria-label=\"Permalink: Timeline\" href=\"#timeline\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eGiven the updates and the times mentioned in the postmortem, we can assemble this timeline:\u003c/p\u003e\n\u003cmarkdown-accessiblity-table\u003e\u003ctable\u003e\n\u003cthead\u003e\n\u003ctr\u003e\n\u003cth align=\"right\"\u003eTime (Pacific)\u003c/th\u003e\n\u003cth\u003eDescription\u003c/th\u003e\n\u003c/tr\u003e\n\u003c/thead\u003e\n\u003ctbody\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e14:50\u003c/td\u003e\n\u003ctd\u003eBenign configuration change released by GCE engineers.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e18:14\u003c/td\u003e\n\u003ctd\u003ePostmortem states that Cloud VPN impact started now.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e18:25\u003c/td\u003e\n\u003ctd\u003eStatus post reports that Cloud VPN impact started now.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e18:51\u003c/td\u003e\n\u003ctd\u003eInitial status post reports issue with Cloud VPN in asia-east1.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:00\u003c/td\u003e\n\u003ctd\u003eSecond status post states impact started at 18:25.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:07\u003c/td\u003e\n\u003ctd\u003eLast edge router stops announcing GCE IP blocks.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:08\u003c/td\u003e\n\u003ctd\u003eInternal alerts generated due to significant traffic loss.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:09\u003c/td\u003e\n\u003ctd\u003eEngineers responding to asia-east1 issue reverted the network configuration change.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:21\u003c/td\u003e\n\u003ctd\u003eStatus post indicates severe network connectivity in all regions.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:27\u003c/td\u003e\n\u003ctd\u003eImpact resolved as per subsequent status post. Postmortem agrees.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd align=\"right\"\u003e19:45\u003c/td\u003e\n\u003ctd\u003eStatus posting at this time indicates impact resolved at 19:27.\u003c/td\u003e\n\u003c/tr\u003e\n\u003c/tbody\u003e\n\u003c/table\u003e\u003c/markdown-accessiblity-table\u003e\n\u003cp dir=\"auto\"\u003eFrom this timeline, we can see that impact started earlier than the status posting indicated. The postmortem indicates that engineers began responding to the asia-east1 issues at 18:14, but the first status post was made at 18:51, a lag of \u003cstrong\u003e37 minutes\u003c/strong\u003e. Additionally, there was a lag of \u003cstrong\u003e13 minutes\u003c/strong\u003e between the time that engineers became aware of total connectivity failure and an update to that effect was posted.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRemediation / Future Work\u003c/h2\u003e\u003ca id=\"user-content-remediation--future-work\" class=\"anchor\" aria-label=\"Permalink: Remediation / Future Work\" href=\"#remediation--future-work\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIn the postmortem, Google stated that they had identified 14 distinct remediations in prevention, detection, and mitigation. They also indicated that they are seeking review by engineers elsewhere in Google that may yield further remediation items. Three of the 14 remediation items were explicitly listed in the postmortem.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAdditional thoughts\u003c/h2\u003e\u003ca id=\"user-content-additional-thoughts\" class=\"anchor\" aria-label=\"Permalink: Additional thoughts\" href=\"#additional-thoughts\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis is the best postmortem I've seen in a long time. It starts and ends with sincere apologies that leave me feeling that Google truly understands and regrets the impact caused by the incident. This is underscored by the following passage:\u003c/p\u003e\n\u003cblockquote\u003e\n\u003cp dir=\"auto\"\u003e[...] we are offering GCE and VPN service credits to all impacted GCP applications equal to (respectively) 10% and 25% of their monthly charges for GCE and VPN. These credits exceed what we promise in the Compute Engine Service Level Agreement (\u003ca href=\"https://cloud.google.com/compute/sla\" rel=\"nofollow\"\u003ehttps://cloud.google.com/compute/sla\u003c/a\u003e) or Cloud VPN Service Level Agreement (\u003ca href=\"https://cloud.google.com/vpn/sla\" rel=\"nofollow\"\u003ehttps://cloud.google.com/vpn/sla\u003c/a\u003e), but are in keeping with the spirit of those SLAs and our ongoing intention to provide a highly-available Google Cloud product suite to all our customers.\u003c/p\u003e\n\u003c/blockquote\u003e\n\u003cp dir=\"auto\"\u003eThe postmortem gives a concise, detailed explanation of impact with exact times. It explains just enough about BGP for an unknowledgeable reader to understand the root cause, while avoiding overcomplicating the postmortem. It includes enough detail that readers familiar with BGP will understand exactly what went wrong and why. Finally, it avoids pointing the finger at any one engineer or system in an effort to shed blame for the incident.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe incident itself is cringe-worthy. As is common in any major incident, multiple distinct failures conspired, resulting in an incident despite built-in safeguards:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe (valid) configuration change failed the automatic consistency check due to a race condition. It is unclear whether this failure mode was previously known.\u003c/li\u003e\n\u003cli\u003eThe system should have rejected the configuration after deeming it inconsistent but instead created an empty configuration. This bug was previously unseen.\u003c/li\u003e\n\u003cli\u003eThe configuration failed the subsequent canary test, but this failure was not reported back and the network management system deemed the configuration to have passed. The postmortem implies that this was also a previously unseen bug.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThe system proceeded to deploy the configuration gradually across the entire infrastructure. The Internet's built-in redundancy masked the issue by continuing to route GCE-bound traffic to edge routers that had not yet applied the new configuration. This made detection of the problem much harder.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAfter resolving impact, the incident response continued as engineers attempted to understand enough about the root cause to be sure that it would not recur. The postmortem was released impressively quickly, indicating a high priority placed on incident followup and external communication. This emphasis on followup can also be seen in the high-quality remediation items listed in the postmortem.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe incident itself is rather concerning, as Google mentions in the conclusion of the postmortem. It spanned all regions in GCE, which means that users could not avoid impact simply through a multi-region deployment. Adding additional safeguards and improving detection and response would help, but to truly avoid this kind of problem in the future, Google would need to split the network management layer by region so that erroneous configurations could not cross the region boundary. This would allow connectivity to a region (through other regions) even if all of that region's edge routers stopped announcing all GCE IP blocks, albeit with increased latency.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIn summary, I consider this a model postmortem because it was released very quickly and contains:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eheartfelt, convincing apologies\u003c/li\u003e\n\u003cli\u003edetailed impact description\u003c/li\u003e\n\u003cli\u003ean appropriate level of detail (enough for experienced engineers but not too much for others)\u003c/li\u003e\n\u003cli\u003edetails on the incident response\u003c/li\u003e\n\u003cli\u003ea precise timeline\u003c/li\u003e\n\u003cli\u003eremediation items\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eIt should be noted that there was a lag in public communication during this incident. It was not particularly severe, but the total connectivity outage was only reported shortly before it was resolved. Improvement here would help customers to deal with such severe incidents in the future.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAbout the Author\u003c/h2\u003e\u003ca id=\"user-content-about-the-author\" class=\"anchor\" aria-label=\"Permalink: About the Author\" href=\"#about-the-author\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eMy name is Lex Neva and I am an SRE at \u003ca href=\"https://heroku.com\" rel=\"nofollow\"\u003eHeroku\u003c/a\u003e. You can reach me at \u003ccode\u003elex at sreweekly.com\u003c/code\u003e or @SREWeekly on Twitter. Subscribe to my newsletter, \u003ca href=\"http://sreweekly.com\" rel=\"nofollow\"\u003eSRE Weekly\u003c/a\u003e, via Twitter, RSS, or email.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch6 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003e#outage #network #bug #technical\u003c/h6\u003e\u003ca id=\"user-content-outage-network-bug-technical\" class=\"anchor\" aria-label=\"Permalink: #outage #network #bug #technical\" href=\"#outage-network-bug-technical\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003c/article\u003e","richTextTruncated":false,"renderedFileInfo":null,"symbols":{"timed_out":false,"not_analyzed":false,"symbols":[{"name":"Report: Google Compute Engine Incident #16007: Connectivity issues in all regions","fully_qualified_name":"Report: Google Compute Engine Incident #16007: Connectivity issues in all regions","kind":"section_1","ident_start":2,"ident_end":83,"extent_start":0,"extent_end":8079,"ident_utf16":{"start":{"line_number":0,"utf16_col":2},"end":{"line_number":0,"utf16_col":83}},"extent_utf16":{"start":{"line_number":0,"utf16_col":0},"end":{"line_number":80,"utf16_col":0}}},{"name":"Company Overview","fully_qualified_name":"Company Overview","kind":"section_2","ident_start":175,"ident_end":191,"extent_start":172,"extent_end":325,"ident_utf16":{"start":{"line_number":3,"utf16_col":3},"end":{"line_number":3,"utf16_col":19}},"extent_utf16":{"start":{"line_number":3,"utf16_col":0},"end":{"line_number":6,"utf16_col":0}}},{"name":"Incident Overview","fully_qualified_name":"Incident Overview","kind":"section_2","ident_start":328,"ident_end":345,"extent_start":325,"extent_end":1381,"ident_utf16":{"start":{"line_number":6,"utf16_col":3},"end":{"line_number":6,"utf16_col":20}},"extent_utf16":{"start":{"line_number":6,"utf16_col":0},"end":{"line_number":11,"utf16_col":0}}},{"name":"Incident Updates / Postmortem","fully_qualified_name":"Incident Updates / Postmortem","kind":"section_2","ident_start":1384,"ident_end":1413,"extent_start":1381,"extent_end":3474,"ident_utf16":{"start":{"line_number":11,"utf16_col":3},"end":{"line_number":11,"utf16_col":32}},"extent_utf16":{"start":{"line_number":11,"utf16_col":0},"end":{"line_number":39,"utf16_col":0}}},{"name":"Updates During the Incident","fully_qualified_name":"Updates During the Incident","kind":"section_3","ident_start":1419,"ident_end":1446,"extent_start":1415,"extent_end":1697,"ident_utf16":{"start":{"line_number":13,"utf16_col":4},"end":{"line_number":13,"utf16_col":31}},"extent_utf16":{"start":{"line_number":13,"utf16_col":0},"end":{"line_number":16,"utf16_col":0}}},{"name":"The postmortem","fully_qualified_name":"The postmortem","kind":"section_3","ident_start":1701,"ident_end":1715,"extent_start":1697,"extent_end":2031,"ident_utf16":{"start":{"line_number":16,"utf16_col":4},"end":{"line_number":16,"utf16_col":18}},"extent_utf16":{"start":{"line_number":16,"utf16_col":0},"end":{"line_number":19,"utf16_col":0}}},{"name":"Timeline","fully_qualified_name":"Timeline","kind":"section_3","ident_start":2035,"ident_end":2043,"extent_start":2031,"extent_end":3474,"ident_utf16":{"start":{"line_number":19,"utf16_col":4},"end":{"line_number":19,"utf16_col":12}},"extent_utf16":{"start":{"line_number":19,"utf16_col":0},"end":{"line_number":39,"utf16_col":0}}},{"name":"Remediation / Future Work","fully_qualified_name":"Remediation / Future Work","kind":"section_2","ident_start":3477,"ident_end":3502,"extent_start":3474,"extent_end":3832,"ident_utf16":{"start":{"line_number":39,"utf16_col":3},"end":{"line_number":39,"utf16_col":28}},"extent_utf16":{"start":{"line_number":39,"utf16_col":0},"end":{"line_number":43,"utf16_col":0}}},{"name":"Additional thoughts","fully_qualified_name":"Additional thoughts","kind":"section_2","ident_start":3835,"ident_end":3854,"extent_start":3832,"extent_end":7784,"ident_utf16":{"start":{"line_number":43,"utf16_col":3},"end":{"line_number":43,"utf16_col":22}},"extent_utf16":{"start":{"line_number":43,"utf16_col":0},"end":{"line_number":74,"utf16_col":0}}},{"name":"About the Author","fully_qualified_name":"About the Author","kind":"section_2","ident_start":7787,"ident_end":7803,"extent_start":7784,"extent_end":8079,"ident_utf16":{"start":{"line_number":74,"utf16_col":3},"end":{"line_number":74,"utf16_col":19}},"extent_utf16":{"start":{"line_number":74,"utf16_col":0},"end":{"line_number":80,"utf16_col":0}}},{"name":"\\#outage #network #bug #technical","fully_qualified_name":"\\#outage #network #bug #technical","kind":"section_6","ident_start":8045,"ident_end":8078,"extent_start":8038,"extent_end":8079,"ident_utf16":{"start":{"line_number":79,"utf16_col":7},"end":{"line_number":79,"utf16_col":40}},"extent_utf16":{"start":{"line_number":79,"utf16_col":0},"end":{"line_number":80,"utf16_col":0}}}]}},"codeViewLayoutRoute":{"repo":{"id":52826457,"defaultBranch":"master","name":"Postmortem-Report-Reviews","ownerLogin":"Operations-Incident-Board","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2016-02-29T21:35:00.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/17555870?v=4","public":true,"private":false,"isOrgOwned":true},"currentUser":null,"uploadToken":"2T4kaDGq3IjOqS_LYAWaxlbyZwD_0DzjgT6I0K582ATZBFltGJse0ZCqlGNeqHdNkByGLJavOf1X3lLfJuarWQ","allShortcutsEnabled":false,"treeExpanded":true,"path":"2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","symbolsExpanded":false,"refInfo":{"name":"master","listCacheKey":"v0:1457842276.0","canEdit":false,"currentOid":"35d26a02701031f005ae42e6b59071010cd4febb"},"helpUrl":"https://docs.github.com","githubDevUrl":null},"codeViewFileTreeLayoutRoute":{"fileTree":{"":{"items":[{"name":"0000-charity-welcome-doc.md","path":"0000-charity-welcome-doc.md","contentType":"file"},{"name":"2016-01-28-pshima-github-outage.md","path":"2016-01-28-pshima-github-outage.md","contentType":"file"},{"name":"2016-03-07-gabinante-AWS-EBS-2011.md","path":"2016-03-07-gabinante-AWS-EBS-2011.md","contentType":"file"},{"name":"2016-03-14-gabinante-netflix-streaming-2012-12-24.md","path":"2016-03-14-gabinante-netflix-streaming-2012-12-24.md","contentType":"file"},{"name":"2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","path":"2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","contentType":"file"},{"name":"2016-07-20-pshima-stack-exchange-2016-07-20.md","path":"2016-07-20-pshima-stack-exchange-2016-07-20.md","contentType":"file"},{"name":"LICENSE","path":"LICENSE","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"}],"totalCount":8}},"fileTreeProcessingTime":14.974203000000001,"foldersToFetch":[]},"codeViewBlobLayoutRoute":{"codeLineWrapEnabled":false,"refInfo":{"name":"master","listCacheKey":"v0:1457842276.0","canEdit":false,"refType":"branch","currentOid":"35d26a02701031f005ae42e6b59071010cd4febb","canEditOnDefaultBranch":false,"fileExistsOnDefault":true},"path":"2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","blob":{"copilotSWEAgentEnabled":false,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/Operations-Incident-Board/Postmortem-Report-Reviews/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","displayUrl":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md?raw=true","headerInfo":{"blobSize":"7.89 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"519d6d3","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2FOperations-Incident-Board%2FPostmortem-Report-Reviews%2Fblob%2Fmaster%2F2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","isCSV":false,"isRichtext":true,"lineInfo":{"truncatedLoc":"80","truncatedSloc":"52"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"isIssueTemplate":false,"isDiscussionTemplate":false,"language":"Markdown","languageID":222,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/Operations-Incident-Board/Postmortem-Report-Reviews/blob/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/Operations-Incident-Board/Postmortem-Report-Reviews/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/raw/refs/heads/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md","renderImageOrRaw":false,"shortPath":null,"symbolsEnabled":true,"tabSize":4,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null},"copilotInfo":null,"copilotAccessAllowed":false,"copilotSpacesEnabled":false,"modelsAccessAllowed":false,"modelsRepoIntegrationEnabled":false,"isMarketplaceEnabled":true},"codeViewBlobLayoutRoute.StyledBlob":{"rawLines":["# Report: Google Compute Engine Incident #16007: Connectivity issues in all regions","Link to Postmortem: https://status.cloud.google.com/incident/compute/16007?post-mortem","","## Company Overview","Google is Google. You may have heard of them. Google Compute Engine is an IaaS offering virtual machines with per-minute billing.","","## Incident Overview","Google Compute Engine instances suffered increasingly higher latency in internet traffic, ultimately culminating in an 18-minute total internet connectivity outage. Edge routers in Google's network began to stop sending BGP announcements for all of GCE's IP blocks. As the incorrect configuration spread to more edge routers, traffic began taking increasingly less-optimal routes until finally no GCE IP blocks were announced by any edge router.","","This incident is especially interesting because it was caused by a combination of three bugs in Google's networking management software, at least one of which was previously unknown. The first bug caused a valid configuration to be rejected, and the second bug caused the system to respond to that rejection by removing *all* GCE IP blocks from the configuration. The third bug caused the canary of this erroneous configuration to be accepted despite the failure. Subsequently, the system slowly deployed the erroneous configuration to edge routers, causing the progressive impact.","","## Incident Updates / Postmortem","","### Updates During the Incident","Google initially posted that they were investigating an issue with Cloud VPN in asia-east1. Subsequently, Google reported \"severe network connectivity issues in all regions\", followed fairly quickly by an update indicating resolution of the issue.","","### The postmortem","About 36 hours after the incident, Google released a lengthy postmortem. It explains BGP at a high level and details the confluence of bugs that resulted in the outage. Google also apologizes multiple times and offers a blanket service credit of 10% off GCE (vm) charges and 25% off VPN charges for the month. ","","### Timeline","","Given the updates and the times mentioned in the postmortem, we can assemble this timeline:","","Time (Pacific) | Description","--------------:|------------","14:50 | Benign configuration change released by GCE engineers.","18:14 | Postmortem states that Cloud VPN impact started now.","18:25 | Status post reports that Cloud VPN impact started now.","18:51 | Initial status post reports issue with Cloud VPN in asia-east1.","19:00 | Second status post states impact started at 18:25.","19:07 | Last edge router stops announcing GCE IP blocks.","19:08 | Internal alerts generated due to significant traffic loss.","19:09 | Engineers responding to asia-east1 issue reverted the network configuration change.","19:21 | Status post indicates severe network connectivity in all regions.","19:27 | Impact resolved as per subsequent status post. Postmortem agrees.","19:45 | Status posting at this time indicates impact resolved at 19:27.","","From this timeline, we can see that impact started earlier than the status posting indicated. The postmortem indicates that engineers began responding to the asia-east1 issues at 18:14, but the first status post was made at 18:51, a lag of **37 minutes**. Additionally, there was a lag of **13 minutes** between the time that engineers became aware of total connectivity failure and an update to that effect was posted.","","## Remediation / Future Work","","In the postmortem, Google stated that they had identified 14 distinct remediations in prevention, detection, and mitigation. They also indicated that they are seeking review by engineers elsewhere in Google that may yield further remediation items. Three of the 14 remediation items were explicitly listed in the postmortem.","","## Additional thoughts","","This is the best postmortem I've seen in a long time. It starts and ends with sincere apologies that leave me feeling that Google truly understands and regrets the impact caused by the incident. This is underscored by the following passage:","","\u003e [...] we are offering GCE and VPN service credits to all impacted GCP applications equal to (respectively) 10% and 25% of their monthly charges for GCE and VPN. These credits exceed what we promise in the Compute Engine Service Level Agreement (https://cloud.google.com/compute/sla) or Cloud VPN Service Level Agreement (https://cloud.google.com/vpn/sla), but are in keeping with the spirit of those SLAs and our ongoing intention to provide a highly-available Google Cloud product suite to all our customers.","","The postmortem gives a concise, detailed explanation of impact with exact times. It explains just enough about BGP for an unknowledgeable reader to understand the root cause, while avoiding overcomplicating the postmortem. It includes enough detail that readers familiar with BGP will understand exactly what went wrong and why. Finally, it avoids pointing the finger at any one engineer or system in an effort to shed blame for the incident.","","The incident itself is cringe-worthy. As is common in any major incident, multiple distinct failures conspired, resulting in an incident despite built-in safeguards:","","* The (valid) configuration change failed the automatic consistency check due to a race condition. It is unclear whether this failure mode was previously known.","* The system should have rejected the configuration after deeming it inconsistent but instead created an empty configuration. This bug was previously unseen.","* The configuration failed the subsequent canary test, but this failure was not reported back and the network management system deemed the configuration to have passed. The postmortem implies that this was also a previously unseen bug.","","The system proceeded to deploy the configuration gradually across the entire infrastructure. The Internet's built-in redundancy masked the issue by continuing to route GCE-bound traffic to edge routers that had not yet applied the new configuration. This made detection of the problem much harder.","","After resolving impact, the incident response continued as engineers attempted to understand enough about the root cause to be sure that it would not recur. The postmortem was released impressively quickly, indicating a high priority placed on incident followup and external communication. This emphasis on followup can also be seen in the high-quality remediation items listed in the postmortem.","","The incident itself is rather concerning, as Google mentions in the conclusion of the postmortem. It spanned all regions in GCE, which means that users could not avoid impact simply through a multi-region deployment. Adding additional safeguards and improving detection and response would help, but to truly avoid this kind of problem in the future, Google would need to split the network management layer by region so that erroneous configurations could not cross the region boundary. This would allow connectivity to a region (through other regions) even if all of that region's edge routers stopped announcing all GCE IP blocks, albeit with increased latency.","","In summary, I consider this a model postmortem because it was released very quickly and contains:","","* heartfelt, convincing apologies","* detailed impact description","* an appropriate level of detail (enough for experienced engineers but not too much for others)","* details on the incident response","* a precise timeline","* remediation items","","It should be noted that there was a lag in public communication during this incident. It was not particularly severe, but the total connectivity outage was only reported shortly before it was resolved. Improvement here would help customers to deal with such severe incidents in the future.","","## About the Author","","My name is Lex Neva and I am an SRE at [Heroku](https://heroku.com). You can reach me at `lex at sreweekly.com` or @SREWeekly on Twitter. Subscribe to my newsletter, [SRE Weekly](http://sreweekly.com), via Twitter, RSS, or email.","","","###### \\#outage #network #bug #technical"],"stylingDirectives":[[[0,83,"pl-mh"],[2,83,"pl-en"],[41,47,"pl-s"],[42,47,"pl-corl"]],[[20,86,"pl-corl"]],[],[[0,19,"pl-mh"],[3,19,"pl-en"]],[],[],[[0,20,"pl-mh"],[3,20,"pl-en"]],[],[],[[321,322,"pl-s"],[325,326,"pl-s"]],[],[[0,32,"pl-mh"],[3,32,"pl-en"]],[],[[0,31,"pl-mh"],[4,31,"pl-en"]],[],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[],[[0,12,"pl-mh"],[4,12,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[[241,243,"pl-s"],[253,255,"pl-s"],[291,293,"pl-s"],[303,305,"pl-s"]],[],[[0,28,"pl-mh"],[3,28,"pl-en"]],[],[],[],[[0,22,"pl-mh"],[3,22,"pl-en"]],[],[],[],[[0,511,"pl-ent"],[0,2,"pl-ent"],[2,3,"pl-s"],[6,7,"pl-s"],[247,283,"pl-corl"],[323,355,"pl-corl"]],[[0,0,"pl-ent"]],[],[],[],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[],[],[],[],[],[],[],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[],[],[[0,19,"pl-mh"],[3,19,"pl-en"]],[],[[39,40,"pl-s"],[46,47,"pl-s"],[47,48,"pl-s"],[48,66,"pl-corl"],[66,67,"pl-s"],[89,90,"pl-s"],[90,110,"pl-c1"],[110,111,"pl-s"],[115,125,"pl-s"],[116,125,"pl-corl"],[167,168,"pl-s"],[178,179,"pl-s"],[179,180,"pl-s"],[180,200,"pl-corl"],[200,201,"pl-s"]],[],[],[[0,40,"pl-mh"],[7,40,"pl-en"],[7,9,"pl-c1"]]],"colorizedLines":null}},"title":"Postmortem-Report-Reviews/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md at master · Operations-Incident-Board/Postmortem-Report-Reviews","appPayload":{},"meta":{"title":"Postmortem-Report-Reviews/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md at master · Operations-Incident-Board/Postmortem-Report-Reviews"}}</script>
|
||
<div data-target="react-app.reactRoot"><meta name="github-code-view-meta-stats" id="github-code-view-meta-stats" data-hydrostats="publish"/> <!-- --> <a hidden="" id="code-view-repo-link" href="/Operations-Incident-Board/Postmortem-Report-Reviews" data-discover="true"></a> <div class="d-none"></div><div><div style="--spacing:var(--spacing-none)" class="prc-PageLayout-PageLayoutRoot--KH-d" data-component="SplitPageLayout" data-has-sidebar="true"><div class="prc-PageLayout-SidebarWrapper-kLG4B CopilotSidePanelSidebar-module__SidePanel__L3O0C CopilotSidePanelSidebar-module__HiddenSidePanel__TBRGn" style="--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="end" data-sticky="true" data-responsive-variant="fullscreen"><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-SidebarVerticalDivider-0Rl0V" data-component="PageLayout.VerticalDivider" data-variant="line" data-position="end" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="450" aria-valuemax="768" aria-valuenow="544" aria-valuetext="Pane width 544 pixels" tabindex="0"></div></div><div class="prc-PageLayout-Sidebar-iciWg" data-component="SplitPageLayout.Sidebar" data-resizable="true" style="--spacing:var(--spacing-normal);--pane-min-width:450px;--pane-max-width:768px;--pane-width-custom:544px;--pane-width-size:var(--pane-width-custom);--pane-width:544px"><div class="height-full" data-testid="copilot-code-view-side-panel"><div id="copilot-side-panel-content" class="height-full"></div></div></div></div><div class="prc-PageLayout-PageLayoutWrapper-2BhU2" data-width="full"><div class="prc-PageLayout-PageLayoutContent-BneH9"><div id="repos-file-tree-sidebar" class="CodeViewFileTreeLayout-module__sidebar__n_Aau" tabindex="0"><div class="prc-PageLayout-PaneWrapper-pHPop ReposFileTreePane-module__Pane__rBZpI ReposFileTreePane-module__HideTree__AYZnm ReposFileTreePane-module__HidePane__VHAVt" style="--offset-header:0px;--spacing-row:var(--spacing-none);--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="start" data-sticky="true"><div class="prc-PageLayout-HorizontalDivider-JLVqp prc-PageLayout-PaneHorizontalDivider-9tbnE" data-component="PageLayout.HorizontalDivider" data-variant-regular="none" data-variant-narrow="none" data-position="start" style="--spacing-divider:var(--spacing-none);--spacing:var(--spacing-none)"></div><div class="prc-PageLayout-Pane-AyzHK" data-component="SplitPageLayout.Pane" data-resizable="true" style="--spacing:var(--spacing-none);--pane-min-width:256px;--pane-max-width:calc(100vw - var(--pane-max-width-diff));--pane-width-size:var(--pane-width-large);--pane-width:320px"></div><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-PaneVerticalDivider-le57g" data-component="PageLayout.VerticalDivider" data-variant-narrow="none" data-variant-regular="line" data-variant-wide="line" data-position="start" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="256" aria-valuemax="600" aria-valuenow="320" aria-valuetext="Pane width 320 pixels" tabindex="0"></div></div></div></div><div data-component="SplitPageLayout.Content" class="prc-PageLayout-ContentWrapper-gR9eG"><div class="prc-PageLayout-Content-xWL-A" data-width="full" style="--spacing:var(--spacing-none)"><div class="SharedPageLayout-module__content__IwGAp" data-selector="repos-split-pane-content" id="repos-split-pane-content" tabindex="0"> <!-- --> <div class="container CodeViewHeader-module__Box__JkPOb"><div class="CodeViewHeader-module__StickyHeader__Qn7UN" id="StickyHeader"><div class="CodeViewHeader-module__Box_1__SbNDV"><div class="CodeViewHeader-module__Box_2__TB46f"><div class="react-code-view-header-wrap--narrow CodeViewHeader-module__Box_3__q1zUL"><div class="CodeViewHeader-module__treeToggleWrapper__RQ__9"><h2 class="use-tree-pane-module__Heading__s4QbZ prc-Heading-Heading-MtWFE" data-component="Heading"><button data-component="Button" type="button" aria-label="Expand file tree" data-testid="expand-file-tree-button-mobile" class="prc-Button-ButtonBase-9n-Xk ExpandFileTreeButton-module__Button_1__Svs95" data-loading="false" data-size="medium" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-left" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.78 12.53a.75.75 0 0 1-1.06 0L2.47 8.28a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L4.81 7h7.44a.75.75 0 0 1 0 1.5H4.81l2.97 2.97a.75.75 0 0 1 0 1.06Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Files</span></span></button><button data-component="IconButton" type="button" data-testid="expand-file-tree-button" aria-controls="repos-file-tree" class="prc-Button-ButtonBase-9n-Xk position-relative ExpandFileTreeButton-module__expandButton__hDOcv ExpandFileTreeButton-module__filesButtonBreakpoint__zEvz3 fgColor-muted prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-labelledby="_R_4lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sidebar-collapse" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.823 7.823a.25.25 0 0 1 0 .354l-2.396 2.396A.25.25 0 0 1 4 10.396V5.604a.25.25 0 0 1 .427-.177Z"></path><path d="M1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0ZM1.5 1.75v12.5c0 .138.112.25.25.25H9.5v-13H1.75a.25.25 0 0 0-.25.25ZM11 14.5h3.25a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25H11Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="se" data-component="Tooltip" aria-hidden="true" id="_R_4lla9lik5_">Expand file tree</span><div class="d-none"></div></h2></div><div class="react-code-view-header-mb--narrow mr-2"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="master branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector-wide"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">master</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="react-code-view-header-mb--narrow CodeViewHeader-module__Box_5__MQ0hL"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__lg__Rjz0A"><nav data-testid="breadcrumbs" aria-labelledby="repos-header-breadcrumb-heading" id="repos-header-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="repos-header-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/Operations-Incident-Board/Postmortem-Report-Reviews/tree/master" data-discover="true">Postmortem-Report-Reviews</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__lg__Rjz0A" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__lg__Rjz0A prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="file-name-id">2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lla9lik5_">Copy path</span></div></div></div><div class="react-code-view-header-element--wide"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><div><div class="CodeViewHeader-module__FileResultsList__JDzUy"><span class="d-flex FileResultsList-module__FilesSearchBox__ivVkc TextInput-wrapper prc-components-TextInputWrapper-Hpdqi prc-components-TextInputBaseWrapper-wY-n0" data-no-trailing-action="true" data-component="TextInput" data-leading-visual="true" data-trailing-visual="true" aria-busy="false"><span class="TextInput-icon" id="_R_5ipla9lik5_" aria-hidden="true" data-component="TextInput.LeadingVisual"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg></span><input type="text" aria-label="Go to file" role="combobox" aria-controls="file-results-list" aria-expanded="false" aria-haspopup="dialog" autoCorrect="off" spellCheck="false" placeholder="Go to file" aria-describedby="_R_5ipla9lik5_ _R_5ipla9lik5H1_" data-component="input" class="prc-components-Input-IwWrt" value=""/><span class="TextInput-icon" id="_R_5ipla9lik5H1_" aria-hidden="true" data-component="TextInput.TrailingVisual"></span></span></div><div class="d-none"></div></div><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-wide" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4pla9lik5_" id="_R_14pla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4pla9lik5_">More file actions</span></div></div></div><div class="react-code-view-header-element--narrow"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-narrow" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4tla9lik5_" id="_R_14tla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4tla9lik5_">More file actions</span></div></div></div></div></div></div></div><div class="CodeView-module__contentWrapper__cG2JH"><div class="react-code-view-bottom-padding"><div class="BlobTopBanners-module__Box__v_nvx"></div></div> <div class="d-none"></div><div class="d-flex flex-column border rounded-2 tmp-mb-3 pl-1"><div class="LatestCommit-module__Box__B25ZT"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">Latest commit</h2><div style="width:120px" class="Skeleton Skeleton--text" data-testid="loading"> </div><div class="d-flex flex-shrink-0 gap-2"><div data-testid="latest-commit-details" class="d-none d-sm-flex flex-items-center"></div><div class="d-flex gap-2"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">History</h2><a data-component="LinkButton" href="/Operations-Incident-Board/Postmortem-Report-Reviews/commits/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" class="prc-Button-ButtonBase-9n-Xk d-none d-lg-flex LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><span class="fgColor-default">History</span></span></span></a><div class="d-sm-none"></div><div class="d-flex d-lg-none"><a data-component="LinkButton" aria-label="View commit history for this file." href="/Operations-Incident-Board/Postmortem-Report-Reviews/commits/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby="_R_4mlala9lik5_"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span></span></a><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" role="tooltip" aria-hidden="true" id="_R_4mlala9lik5_">History</span></div></div></div></div></div><div class="d-flex flex-row"><div class="container BlobViewContent-module__blobContainer__DtH2d"><div class="react-code-size-details-banner BlobViewContent-module__codeSizeDetails__e5sUw"><div class="react-code-size-details-banner CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="7.89 KB" style="--truncate-max-width:100%"><span>80 lines (52 loc) · 7.89 KB</span></div></div></div></div><div class="react-blob-view-header-sticky BlobViewContent-module__stickyHeader__VwxB5" id="repos-sticky-header"><div class="BlobViewHeader-module__Box__yhm9u"><div class="react-blob-sticky-header"><div class="FileNameStickyHeader-module__outerWrapper__ZL4Xc FileNameStickyHeader-module__outerWrapperHidden__Zpynk"><div class="FileNameStickyHeader-module__Box_1__Hazu5"><div class="FileNameStickyHeader-module__Box_2__hoolP"><div class="FileNameStickyHeader-module__Box_3__MVKsk"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="master branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">master</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="FileNameStickyHeader-module__Box_4__FLhtt"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__md__Wb1Gs"><nav data-testid="breadcrumbs" aria-labelledby="sticky-breadcrumb-heading" id="sticky-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="sticky-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/Operations-Incident-Board/Postmortem-Report-Reviews/tree/master" data-discover="true">Postmortem-Report-Reviews</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__md__Wb1Gs" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__md__Wb1Gs prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="sticky-file-name-id">2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lcpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lcpala9lik5_">Copy path</span></div></div></div><button data-component="Button" type="button" class="prc-Button-ButtonBase-9n-Xk FileNameStickyHeader-module__Button__LSEU_ FileNameStickyHeader-module__GoToTopButton__nxAFn" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-up" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.47 7.78a.75.75 0 0 1 0-1.06l4.25-4.25a.75.75 0 0 1 1.06 0l4.25 4.25a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018L9 4.81v7.44a.75.75 0 0 1-1.5 0V4.81L4.53 7.78a.75.75 0 0 1-1.06 0Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Top</span></span></button></div></div></div><div class="BlobViewHeader-module__Box_1__VEmuQ"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">File metadata and controls</h2><div class="BlobViewHeader-module__Box_2__icUs2"><ul aria-label="File view" class="prc-SegmentedControl-SegmentedControl-lqIXp BlobTabButtons-module__SegmentedControl__jen2u" data-variant="default" data-size="small" data-component="SegmentedControl"><li class="prc-SegmentedControl-Item-tSCQh" data-selected="" data-component="SegmentedControl.Button"><button aria-pressed="true" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:transparent"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Preview">Preview</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Code">Code</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Blame">Blame</div></span></button></li></ul><div class="d-none"></div><div class="react-code-size-details-in-header CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="7.89 KB" style="--truncate-max-width:100%"><span>80 lines (52 loc) · 7.89 KB</span></div></div></div></div><div class="BlobViewHeader-module__Box_3__ng6v2"><div class="d-none"></div><div class="react-blob-header-edit-and-raw-actions BlobViewHeader-module__Box_4__J4Y4W"><div class="d-none"></div><div class="prc-ButtonGroup-ButtonGroup-vFUrY" data-component="ButtonGroup"><div class="prc-ButtonGroup-Item-PqvDl"><a data-component="LinkButton" href="https://github.com/Operations-Incident-Board/Postmortem-Report-Reviews/raw/refs/heads/master/2016-04-14-lexelby-Google-Compute-Engine-2016-04-11.md" data-testid="raw-button" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov BlobViewHeader-module__LinkButton__X9kx2" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Raw</span></span></a></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="copy-raw-button" class="prc-Button-ButtonBase-9n-Xk prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_qaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_qaucpala9lik5_">Copy raw file</span></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="download-raw-button" class="prc-Button-ButtonBase-9n-Xk BlobViewHeader-module__downloadButton__ef459 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_eaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-download" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2.75 14A1.75 1.75 0 0 1 1 12.25v-2.5a.75.75 0 0 1 1.5 0v2.5c0 .138.112.25.25.25h10.5a.25.25 0 0 0 .25-.25v-2.5a.75.75 0 0 1 1.5 0v2.5A1.75 1.75 0 0 1 13.25 14Z"></path><path d="M7.25 7.689V2a.75.75 0 0 1 1.5 0v5.689l1.97-1.969a.749.749 0 1 1 1.06 1.06l-3.25 3.25a.749.749 0 0 1-1.06 0L4.22 6.78a.749.749 0 1 1 1.06-1.06l1.97 1.969Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_eaucpala9lik5_">Download raw file</span></div></div></div><button data-component="IconButton" type="button" aria-pressed="false" class="prc-Button-ButtonBase-9n-Xk tmp-mr-2 TableOfContents-module__IconButton__jrlNM prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_3ucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-list-unordered" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.75 2.5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2 14a1 1 0 1 1 0-2 1 1 0 0 1 0 2Zm1-6a1 1 0 1 1-2 0 1 1 0 0 1 2 0ZM2 4a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_3ucpala9lik5_">Outline</span><div class="react-blob-header-edit-and-raw-actions-combined"><button data-component="IconButton" type="button" title="More file actions" data-testid="more-file-actions-button" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click BlobViewHeader-module__IconButton__XrMQY prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_fkecpala9lik5_" id="_R_kecpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_fkecpala9lik5_">Edit and raw actions</span></div></div></div></div><div></div></div><div class="BlobViewContent-module__blobContentWrapper__JS0W6"><section aria-labelledby="file-name-id-wide file-name-id-mobile" class="BlobContent-module__blobContentSection__VOgZq BlobContent-module__blobContentSectionMarkdown__mPLOK" style="margin-top:46px"><div class="js-snippet-clipboard-copy-unpositioned BlobContent-module__markdownBlob__T8jpG" data-hpc="true" containertiming="hpc"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Report: Google Compute Engine Incident #16007: Connectivity issues in all regions</h1><a id="user-content-report-google-compute-engine-incident-16007-connectivity-issues-in-all-regions" class="anchor" aria-label="Permalink: Report: Google Compute Engine Incident #16007: Connectivity issues in all regions" href="#report-google-compute-engine-incident-16007-connectivity-issues-in-all-regions"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Link to Postmortem: <a href="https://status.cloud.google.com/incident/compute/16007?post-mortem" rel="nofollow">https://status.cloud.google.com/incident/compute/16007?post-mortem</a></p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Company Overview</h2><a id="user-content-company-overview" class="anchor" aria-label="Permalink: Company Overview" href="#company-overview"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Google is Google. You may have heard of them. Google Compute Engine is an IaaS offering virtual machines with per-minute billing.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Incident Overview</h2><a id="user-content-incident-overview" class="anchor" aria-label="Permalink: Incident Overview" href="#incident-overview"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Google Compute Engine instances suffered increasingly higher latency in internet traffic, ultimately culminating in an 18-minute total internet connectivity outage. Edge routers in Google's network began to stop sending BGP announcements for all of GCE's IP blocks. As the incorrect configuration spread to more edge routers, traffic began taking increasingly less-optimal routes until finally no GCE IP blocks were announced by any edge router.</p>
|
||
<p dir="auto">This incident is especially interesting because it was caused by a combination of three bugs in Google's networking management software, at least one of which was previously unknown. The first bug caused a valid configuration to be rejected, and the second bug caused the system to respond to that rejection by removing <em>all</em> GCE IP blocks from the configuration. The third bug caused the canary of this erroneous configuration to be accepted despite the failure. Subsequently, the system slowly deployed the erroneous configuration to edge routers, causing the progressive impact.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Incident Updates / Postmortem</h2><a id="user-content-incident-updates--postmortem" class="anchor" aria-label="Permalink: Incident Updates / Postmortem" href="#incident-updates--postmortem"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Updates During the Incident</h3><a id="user-content-updates-during-the-incident" class="anchor" aria-label="Permalink: Updates During the Incident" href="#updates-during-the-incident"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Google initially posted that they were investigating an issue with Cloud VPN in asia-east1. Subsequently, Google reported "severe network connectivity issues in all regions", followed fairly quickly by an update indicating resolution of the issue.</p>
|
||
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">The postmortem</h3><a id="user-content-the-postmortem" class="anchor" aria-label="Permalink: The postmortem" href="#the-postmortem"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">About 36 hours after the incident, Google released a lengthy postmortem. It explains BGP at a high level and details the confluence of bugs that resulted in the outage. Google also apologizes multiple times and offers a blanket service credit of 10% off GCE (vm) charges and 25% off VPN charges for the month.</p>
|
||
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Timeline</h3><a id="user-content-timeline" class="anchor" aria-label="Permalink: Timeline" href="#timeline"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Given the updates and the times mentioned in the postmortem, we can assemble this timeline:</p>
|
||
<markdown-accessiblity-table><table>
|
||
<thead>
|
||
<tr>
|
||
<th align="right">Time (Pacific)</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td align="right">14:50</td>
|
||
<td>Benign configuration change released by GCE engineers.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">18:14</td>
|
||
<td>Postmortem states that Cloud VPN impact started now.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">18:25</td>
|
||
<td>Status post reports that Cloud VPN impact started now.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">18:51</td>
|
||
<td>Initial status post reports issue with Cloud VPN in asia-east1.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:00</td>
|
||
<td>Second status post states impact started at 18:25.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:07</td>
|
||
<td>Last edge router stops announcing GCE IP blocks.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:08</td>
|
||
<td>Internal alerts generated due to significant traffic loss.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:09</td>
|
||
<td>Engineers responding to asia-east1 issue reverted the network configuration change.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:21</td>
|
||
<td>Status post indicates severe network connectivity in all regions.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:27</td>
|
||
<td>Impact resolved as per subsequent status post. Postmortem agrees.</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="right">19:45</td>
|
||
<td>Status posting at this time indicates impact resolved at 19:27.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table></markdown-accessiblity-table>
|
||
<p dir="auto">From this timeline, we can see that impact started earlier than the status posting indicated. The postmortem indicates that engineers began responding to the asia-east1 issues at 18:14, but the first status post was made at 18:51, a lag of <strong>37 minutes</strong>. Additionally, there was a lag of <strong>13 minutes</strong> between the time that engineers became aware of total connectivity failure and an update to that effect was posted.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Remediation / Future Work</h2><a id="user-content-remediation--future-work" class="anchor" aria-label="Permalink: Remediation / Future Work" href="#remediation--future-work"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">In the postmortem, Google stated that they had identified 14 distinct remediations in prevention, detection, and mitigation. They also indicated that they are seeking review by engineers elsewhere in Google that may yield further remediation items. Three of the 14 remediation items were explicitly listed in the postmortem.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Additional thoughts</h2><a id="user-content-additional-thoughts" class="anchor" aria-label="Permalink: Additional thoughts" href="#additional-thoughts"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">This is the best postmortem I've seen in a long time. It starts and ends with sincere apologies that leave me feeling that Google truly understands and regrets the impact caused by the incident. This is underscored by the following passage:</p>
|
||
<blockquote>
|
||
<p dir="auto">[...] we are offering GCE and VPN service credits to all impacted GCP applications equal to (respectively) 10% and 25% of their monthly charges for GCE and VPN. These credits exceed what we promise in the Compute Engine Service Level Agreement (<a href="https://cloud.google.com/compute/sla" rel="nofollow">https://cloud.google.com/compute/sla</a>) or Cloud VPN Service Level Agreement (<a href="https://cloud.google.com/vpn/sla" rel="nofollow">https://cloud.google.com/vpn/sla</a>), but are in keeping with the spirit of those SLAs and our ongoing intention to provide a highly-available Google Cloud product suite to all our customers.</p>
|
||
</blockquote>
|
||
<p dir="auto">The postmortem gives a concise, detailed explanation of impact with exact times. It explains just enough about BGP for an unknowledgeable reader to understand the root cause, while avoiding overcomplicating the postmortem. It includes enough detail that readers familiar with BGP will understand exactly what went wrong and why. Finally, it avoids pointing the finger at any one engineer or system in an effort to shed blame for the incident.</p>
|
||
<p dir="auto">The incident itself is cringe-worthy. As is common in any major incident, multiple distinct failures conspired, resulting in an incident despite built-in safeguards:</p>
|
||
<ul dir="auto">
|
||
<li>The (valid) configuration change failed the automatic consistency check due to a race condition. It is unclear whether this failure mode was previously known.</li>
|
||
<li>The system should have rejected the configuration after deeming it inconsistent but instead created an empty configuration. This bug was previously unseen.</li>
|
||
<li>The configuration failed the subsequent canary test, but this failure was not reported back and the network management system deemed the configuration to have passed. The postmortem implies that this was also a previously unseen bug.</li>
|
||
</ul>
|
||
<p dir="auto">The system proceeded to deploy the configuration gradually across the entire infrastructure. The Internet's built-in redundancy masked the issue by continuing to route GCE-bound traffic to edge routers that had not yet applied the new configuration. This made detection of the problem much harder.</p>
|
||
<p dir="auto">After resolving impact, the incident response continued as engineers attempted to understand enough about the root cause to be sure that it would not recur. The postmortem was released impressively quickly, indicating a high priority placed on incident followup and external communication. This emphasis on followup can also be seen in the high-quality remediation items listed in the postmortem.</p>
|
||
<p dir="auto">The incident itself is rather concerning, as Google mentions in the conclusion of the postmortem. It spanned all regions in GCE, which means that users could not avoid impact simply through a multi-region deployment. Adding additional safeguards and improving detection and response would help, but to truly avoid this kind of problem in the future, Google would need to split the network management layer by region so that erroneous configurations could not cross the region boundary. This would allow connectivity to a region (through other regions) even if all of that region's edge routers stopped announcing all GCE IP blocks, albeit with increased latency.</p>
|
||
<p dir="auto">In summary, I consider this a model postmortem because it was released very quickly and contains:</p>
|
||
<ul dir="auto">
|
||
<li>heartfelt, convincing apologies</li>
|
||
<li>detailed impact description</li>
|
||
<li>an appropriate level of detail (enough for experienced engineers but not too much for others)</li>
|
||
<li>details on the incident response</li>
|
||
<li>a precise timeline</li>
|
||
<li>remediation items</li>
|
||
</ul>
|
||
<p dir="auto">It should be noted that there was a lag in public communication during this incident. It was not particularly severe, but the total connectivity outage was only reported shortly before it was resolved. Improvement here would help customers to deal with such severe incidents in the future.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">About the Author</h2><a id="user-content-about-the-author" class="anchor" aria-label="Permalink: About the Author" href="#about-the-author"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">My name is Lex Neva and I am an SRE at <a href="https://heroku.com" rel="nofollow">Heroku</a>. You can reach me at <code>lex at sreweekly.com</code> or @SREWeekly on Twitter. Subscribe to my newsletter, <a href="http://sreweekly.com" rel="nofollow">SRE Weekly</a>, via Twitter, RSS, or email.</p>
|
||
<div class="markdown-heading" dir="auto"><h6 tabindex="-1" class="heading-element" dir="auto">#outage #network #bug #technical</h6><a id="user-content-outage-network-bug-technical" class="anchor" aria-label="Permalink: #outage #network #bug #technical" href="#outage-network-bug-technical"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
</article></div><div class="d-none"></div></section></div></div></div> </div> <!-- --> </div></div></div></div></div></div><div class="ScrollMarksContainer-module__scrollMarksContainer__Eu7uU" id="find-result-marks-container"></div><div class="d-none"></div><div class="d-none"></div></div> <!-- --> <!-- --> </div>
|
||
</react-app>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</turbo-frame>
|
||
|
||
</main>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<footer class="footer f6 color-fg-muted color-border-subtle tmp-pt-7 tmp-pb-6 p-responsive" role="contentinfo" >
|
||
<h2 class='sr-only'>Footer</h2>
|
||
|
||
|
||
|
||
|
||
<div class="d-flex flex-justify-center flex-items-center flex-column-reverse flex-lg-row flex-wrap flex-lg-nowrap">
|
||
<div class="d-flex flex-items-center flex-shrink-0 mx-2">
|
||
<a aria-label="GitHub Homepage" class="footer-octicon mr-2" href="https://github.com">
|
||
<svg aria-hidden="true" data-component="Octicon" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-mark-github">
|
||
<path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path>
|
||
</svg>
|
||
</a>
|
||
<span>
|
||
© 2026 GitHub, Inc.
|
||
</span>
|
||
</div>
|
||
|
||
<nav aria-label="Footer">
|
||
<h3 class="sr-only" id="sr-footer-heading">Footer navigation</h3>
|
||
|
||
<ul class="list-style-none d-flex flex-justify-center flex-wrap mb-2 mb-lg-0" aria-labelledby="sr-footer-heading">
|
||
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to Terms","label":"text:terms"}" href="https://docs.github.com/site-policy/github-terms/github-terms-of-service" data-view-component="true" class="Link--secondary Link">Terms</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to privacy","label":"text:privacy"}" href="https://docs.github.com/site-policy/privacy-policies/github-privacy-statement" data-view-component="true" class="Link--secondary Link">Privacy</a>
|
||
</li>
|
||
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to security","label":"text:security"}" href="https://github.com/security" data-view-component="true" class="Link--secondary Link">Security</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to status","label":"text:status"}" href="https://www.githubstatus.com/" data-view-component="true" class="Link--secondary Link">Status</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to community","label":"text:community"}" href="https://github.community/" data-view-component="true" class="Link--secondary Link">Community</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to docs","label":"text:docs"}" href="https://docs.github.com/" data-view-component="true" class="Link--secondary Link">Docs</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to contact","label":"text:contact"}" href="https://support.github.com?tags=dotcom-footer" data-view-component="true" class="Link--secondary Link">Contact</a>
|
||
</li>
|
||
|
||
|
||
<li class="mx-2" >
|
||
<cookie-consent-link>
|
||
<button
|
||
type="button"
|
||
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent"
|
||
data-action="click:cookie-consent-link#showConsentManagement"
|
||
data-analytics-event="{"location":"footer","action":"cookies","context":"subfooter","tag":"link","label":"cookies_link_subfooter_footer"}"
|
||
>
|
||
Manage cookies
|
||
</button>
|
||
</cookie-consent-link>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<cookie-consent-link>
|
||
<button
|
||
type="button"
|
||
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent text-left"
|
||
data-action="click:cookie-consent-link#showConsentManagement"
|
||
data-analytics-event="{"location":"footer","action":"dont_share_info","context":"subfooter","tag":"link","label":"dont_share_info_link_subfooter_footer"}"
|
||
>
|
||
Do not share my personal information
|
||
</button>
|
||
</cookie-consent-link>
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
|
||
<ghcc-consent id="ghcc" class="position-fixed bottom-0 left-0" style="z-index: 999999"
|
||
data-locale="en"
|
||
data-initial-cookie-consent-allowed=""
|
||
data-cookie-consent-required="false"
|
||
></ghcc-consent>
|
||
|
||
|
||
|
||
|
||
<div id="ajax-error-message" class="ajax-error-message flash flash-error" hidden>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
|
||
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<button type="button" class="flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
You can’t perform that action at this time.
|
||
</div>
|
||
|
||
<template id="site-details-dialog">
|
||
<details class="details-reset details-overlay details-overlay-dark lh-default color-fg-default hx_rsm" open>
|
||
<summary role="button" aria-label="Close dialog"></summary>
|
||
<details-dialog class="Box Box--overlay d-flex flex-column anim-fade-in fast hx_rsm-dialog hx_rsm-modal">
|
||
<button class="Box-btn-octicon m-0 btn-octicon position-absolute right-0 top-0" type="button" aria-label="Close dialog" data-close-dialog>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
<div class="octocat-spinner tmp-my-6 js-details-dialog-spinner"></div>
|
||
</details-dialog>
|
||
</details>
|
||
</template>
|
||
|
||
<div class="Popover js-hovercard-content position-absolute" style="display: none; outline: none;">
|
||
<div class="Popover-message Popover-message--bottom-left Popover-message--large Box color-shadow-large" style="width:360px;">
|
||
</div>
|
||
</div>
|
||
|
||
<template id="snippet-clipboard-copy-button">
|
||
<div class="zeroclipboard-container position-absolute right-0 top-0">
|
||
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn js-clipboard-copy m-2 p-0" data-copy-feedback="Copied!" data-tooltip-direction="w">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon m-2 tmp-m-2">
|
||
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
|
||
</svg>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none m-2 tmp-m-2">
|
||
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
|
||
</svg>
|
||
</clipboard-copy>
|
||
</div>
|
||
</template>
|
||
<template id="snippet-clipboard-copy-button-unpositioned">
|
||
<div class="zeroclipboard-container">
|
||
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon">
|
||
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
|
||
</svg>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none">
|
||
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
|
||
</svg>
|
||
</clipboard-copy>
|
||
</div>
|
||
</template>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
<div id="js-global-screen-reader-notice" class="sr-only mt-n1" aria-live="polite" aria-atomic="true" ></div>
|
||
<div id="js-global-screen-reader-notice-assertive" class="sr-only mt-n1" aria-live="assertive" aria-atomic="true"></div>
|
||
</body>
|
||
</html>
|
||
|