1183 lines
364 KiB
HTML
1183 lines
364 KiB
HTML
|
||
|
||
|
||
|
||
|
||
|
||
<!DOCTYPE html>
|
||
<html
|
||
lang="en"
|
||
|
||
data-color-mode="auto" data-light-theme="light" data-dark-theme="dark"
|
||
data-a11y-animated-images="system" data-a11y-link-underlines="true"
|
||
|
||
>
|
||
|
||
|
||
|
||
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<link rel="dns-prefetch" href="https://github.githubassets.com">
|
||
<link rel="dns-prefetch" href="https://avatars.githubusercontent.com">
|
||
<link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
|
||
<link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
|
||
<link rel="preconnect" href="https://github.githubassets.com" crossorigin>
|
||
<link rel="preconnect" href="https://avatars.githubusercontent.com">
|
||
|
||
<script type="importmap">{"imports":{"react":"https://github.githubassets.com/assets/react-e27d1b3e03961e68.js","react-dom":"https://github.githubassets.com/assets/react-dom-e5fd46a22d5c4058.js","react-dom/client":"https://github.githubassets.com/assets/react-dom-client-1b4a3ee065998cea.js","react-is":"https://github.githubassets.com/assets/react-is-e0b593954b4706d8.js","react-reconciler":"https://github.githubassets.com/assets/react-reconciler-8e99e505c4429605.js","react/compiler-runtime":"https://github.githubassets.com/assets/react-compiler-runtime-4610bd6d3de9c049.js","react/jsx-dev-runtime":"https://github.githubassets.com/assets/react-jsx-dev-runtime-ea55d68667d559e5.js","react/jsx-runtime":"https://github.githubassets.com/assets/react-jsx-runtime-4915cb0f5b3aff04.js","scheduler":"https://github.githubassets.com/assets/scheduler-58b860b049ca307c.js"}}</script>
|
||
<meta name="react-profiling" content="0" data-turbo-transient="true" />
|
||
<meta name="react-import-map" content="react,react-dom,react-dom/client,react-dom/profiling,react-is,react-reconciler,react/compiler-runtime,react/jsx-dev-runtime,react/jsx-runtime,scheduler@777f63f87cd0" data-turbo-track="reload" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-e27d1b3e03961e68.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-compiler-runtime-4610bd6d3de9c049.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/scheduler-58b860b049ca307c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-dom-e5fd46a22d5c4058.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-dom-client-1b4a3ee065998cea.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-is-e0b593954b4706d8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-jsx-runtime-4915cb0f5b3aff04.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-reconciler-8e99e505c4429605.js" />
|
||
|
||
|
||
|
||
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="light" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-f4bf1142976e4bbf.css" /><link data-color-theme="light_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind_high_contrast-f661b49995ba0bd8.css" /><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-0b38d22346321c92.css" /><link data-color-theme="light_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia_high_contrast-f1c62c9e70259b9f.css" /><link data-color-theme="dark" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-f50cacf0a86b9929.css" /><link data-color-theme="dark_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind_high_contrast-e61d4f4ca17852c2.css" /><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-39c10993d5603fac.css" /><link data-color-theme="dark_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia_high_contrast-73236c840c0c7d90.css" /><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-0de76f07cc035b10.css" /><link data-color-theme="dark_dimmed_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed_high_contrast-fd1500c8744e40d6.css" />
|
||
|
||
<style type="text/css">
|
||
:root {
|
||
--tab-size-preference: 4;
|
||
}
|
||
|
||
pre, code {
|
||
tab-size: var(--tab-size-preference);
|
||
}
|
||
</style>
|
||
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-ed9ca172356fd545.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-1d2c7f7b52a6068b.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-adcabba7b5c5d221.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-552513ad07a183a1.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repository-11ee8a031c040c1a.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-2d56bdb0166c0238.css" />
|
||
|
||
|
||
|
||
<script type="application/json" id="client-env">{"locale":"en","featureFlags":["actions_enable_background_steps","actions_new_hosted_runner_image_select_sizes_and_versions","activity_diff_file_tree","activity_repos_file_tree","activity_repos_overview_header","activity_repos_overview_sidebar","agent_author_search_expansion","agent_author_search_expansion_ui_pulls","alternate_user_config_repo","billing_billable_licenses_cost_center_bucket_fix","billing_budget_expiration","billing_cost_center_list_assigned_resources","billing_discount_threshold_notification","cca_show_aic_usage_per_session","ccr_files_changed_model_picker","code_quality_enablement_banner_targeting","code_quality_remove_preview","code_view_raf_sticky_lines","codespaces_prebuild_region_target_update","coding_agent_third_party_model_ui","copilot_agent_snippy","copilot_api_agentic_issue_marshal_yaml","copilot_automations_pagination","copilot_chat_clear_model_selection_for_default_change","copilot_chat_compact_tables","copilot_chat_header_reorder","copilot_chat_max_upsell","copilot_chat_minimize_contextual","copilot_chat_model_picker_promotions","copilot_chat_models_browser_cache","copilot_chat_new_topic_nudge","copilot_chat_reduce_quota_checks","copilot_chat_vision_dotcom_chat_ga_gate","copilot_chat_vision_preview_gate","copilot_css_textarea_autosize","copilot_custom_copilots","copilot_custom_copilots_feature_preview","copilot_diff_reference_context","copilot_duplicate_thread","copilot_extensions_removal_on_marketplace","copilot_fix_failed_workflows_all_skus","copilot_ftp_hyperspace_upgrade_prompt","copilot_hide_hovercard","copilot_immersive_code_block_transition_wrap","copilot_immersive_embedded_draggable","copilot_immersive_embedded_header_button","copilot_immersive_file_block_transition_open","copilot_immersive_file_preview_keep_mounted","copilot_immersive_suggestion_pills","copilot_immersive_task_hyperlinking","copilot_immersive_task_within_chat_thread","copilot_mc_cli_resume_any_users_task","copilot_mission_control_agent_merge_fix_ci","copilot_mission_control_agent_merge_resolve_conflicts","copilot_mission_control_early_stop","copilot_mission_control_environment_list_icons","copilot_mission_control_managed_sandbox_environments","copilot_mission_control_needs_attention","copilot_mission_control_reasoning_effort","copilot_mission_control_repo_hovercards","copilot_mission_control_sandbox_client_side_clone","copilot_mission_control_sandbox_remote_bypass","copilot_mission_control_session_filters","copilot_mission_control_task_alive_updates","copilot_mission_control_task_sharing","copilot_org_policy_page_focus_mode","copilot_pr_chat_enhancements","copilot_prominent_upgrade_button","copilot_resource_panel","copilot_share_active_subthread","copilot_spaces_ga","copilot_spaces_individual_policies_ga","copilot_spark_handle_nil_friendly_name","copilot_swe_agent_authorization_status_ui","copilot_swe_agent_automation_resource_scoped_writes","copilot_swe_agent_hide_model_picker_if_only_auto","copilot_swe_agent_issue_comment_trigger","copilot_swe_agent_pr_comment_model_picker","copilot_swe_agent_pull_request_comment_trigger","copilot_swe_agent_pull_request_merged_trigger","copilot_swe_agent_pull_request_opened_trigger","copilot_swe_agent_pull_request_synchronize_trigger","copilot_swe_agent_use_subagents","copilot_task_api_github_rest_style","copilot_task_scoped_alive_channel","copilot_token_based_billing","copilot_unconfigured_is_inherited","copilot_user_can_upgrade_plan_field","copilot_workbench_sunset","copilot_workbench_sunset_redirect","copilot_workbench_ubb","dashboard_indexeddb_caching","dashboard_lists_max_age_filter","dashboard_universe_2025_feedback_dialog","flex_suite_disable_river_accordion_dither","glc_code_quality_repo_settings_workflow_config","hide_github_models_ui","hyperspace_2025_logged_out_batch_1","hyperspace_2025_logged_out_batch_2","hyperspace_2025_logged_out_batch_3","in_product_messaging_datadog_monitoring","ipm_global_transactional_message_copilot","ipm_global_transactional_message_issues","ipm_global_transactional_message_prs","ipm_global_transactional_message_repos","ipm_global_transactional_message_spaces","issue_fields_multi_select","issue_inline_avatars","issue_pinned_views","issue_pinned_views_optimistic_updates","issue_relative_time_micro","issues_dashboard_sso_structured_errors","issues_expanded_file_types","issues_hide_closed_sub_issues","issues_lazy_load_comment_box_suggestions","issues_react_chrome_container_query_fix","labels_archiving","labels_archiving_info","landing_pages_ninetailed","lifecycle_label_name_updates","marketing_pages_search_explore_provider","memex_default_issue_create_repository","memex_lazy_hydrate_agent_tasks","memex_live_update_hovercard","memex_mwl_filter_field_delimiter","memex_remove_deprecated_type_issue","merge_queue_restricted_pushers_warning","merge_status_checks_refetch_dedupe","merge_status_header_feedback","oauth_authorize_clickjacking_protection","octocaptcha_origin_optimization","primer_react_css_anchor_positioning","primer_react_merged_forwarded_refs","prs_copilot_app_open_action","prs_css_anchor_positioning","prs_new_conversation_comments_api","pull_request_copilot_attribution_header","pull_request_overview_panel_edit_description","pull_request_persister","pull_request_stacks_feedback_dialog","pull_request_stacks_rebase_conflict_instructions","pull_request_virtualization_image_estimate","pull_request_virtualization_scroll_compensation","pull_request_virtualization_scroll_intent","pulls_dashboard_sidebar_layout","quick_search_lazy_suggestions","react_blob_isolate_code_lines","react_blob_ssr_content_visibility","react_data_router_tanstack_allowed","react_query_props_with_key","react_sandbox_future_tanstack","repo_app_actions_workflows","repo_app_agents","repo_app_commits","repo_app_issues","repo_app_pull_requests","repo_app_turbo","repo_issues_sidebar_layout","repo_overview_ask_copilot","repo_pulls_dashboard_persistence","repos_contributors_limited_default_range","review_involves_filter","rule_ignored_file_paths","rulesets_actor_list_editor","sample_network_conn_type","security_center_artifact_filters_popover","see_who_reacted","semantic_similarity_duplicate_issue_detection","session_logs_ungroup_reasoning_text","set_sha256_on_repo_creation_form","site_banner_desktop_copilot_app","site_ghca_pixel_mona","site_github_app_ga_page","site_github_app_ga_page_highlight","site_github_app_mobile_native_share","site_global_banner_dev_days_attendee","site_global_banner_learn_copilot_sdk","site_global_nav_spark_models_removed","spark_prompt_secret_scanning","spark_server_connection_status","suppress_automated_browser_vitals","swp_forms_disable_octocaptcha","thread_resolution_reason","ui_service_referrer_metrics","update_issue_suggestions","viewscreen_sandbox","warn_inaccessible_attachments","webp_support","workbench_store_readonly","react_import_map","react_import_map"],"githubDomain":"https://github.com","copilotApiOverrideUrl":"https://api.githubcopilot.com","cmcApiUrl":"https://api.github.com/cmc_internal/api"}</script>
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/high-contrast-cookie-e3d808ee18eb9784.js"></script>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wp-runtime-c40ff75fa4b29faf.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-foundation-89338d13d121427c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-runtime-79275629e212a8de.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fetch-utilities-ef80792c471f3488.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ser-564408e9198dff6d.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/environment-6301446d495cadb7.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/app-runtime.db39abd7070fb379.module.css" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/catalyst-86a8f5de995615f8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/selector-observer-e88088f989b27670.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/relative-time-element-85cb37305f54dec4.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/by4-73006a52f155289f.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ja9-36f036c13c2e3a4c.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/kw-866fd9513ae95106.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/r1-b075aab3204f32f9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/jz5-7c5d1669717041f8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hj-5126390ff433704d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j05-dce4d588268f0f7e.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/github-elements-d5a098151cf3b06a.js" defer="defer"></script>
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/element-registry-e87c875160a83f53.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/runtime-helpers-f3f5d71440009c48.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aria-live-edd4f18194230d9e.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hotkey-5109c77d7ac61078.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-core-220b14bbb2e5a3aa.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/04-a76c47ec12c12e01.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ur-024971cb63acad59.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4t-9933bcccf9a9524f.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/6ny-c969fbae697a356f.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/yhq-3a209035a30b1a09.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/z1v-da5b33b877bfa358.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wo-09f85d22a6c8931b.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/9y-81cfe57dc9040f3e.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j0-41a4cf04cbd99503.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0p-50de93b282dbc67c.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/behaviors-f18992dbb9345e7b.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/react-core.1d66df2418934e22.module.css" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-menu-e333ef3cca1b9526.js" defer="defer"></script>
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/primer-react-7450ad822d7699b5.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-lib-3b03eb55bae46c70.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aq-ba5f008999aedbd0.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j6-363f0ebf4f62f310.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/t7l-bf6f21d7e7a43c35.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fe-7576d9b281e2106a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wsr-5fd3071e5a9061ab.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0n-27006551de0ea3c6.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xv-51cb34754bda73a9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/n4u-6317ce1c31aed1db.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j8s-d05eedc69cebc200.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/la-de176c4ee471f2b3.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/pj-da0fc66367c63559.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/oo-ed8e4c10812fc721.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/50-3e117c4737ded7c8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/5gq-0ecd117c4ccaf4f0.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nsi-926ffbc13301c5b7.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3np-a0186ba63220b6ce.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/on-75d8015f7c16be87.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bi0-e2f78de823e39ca9.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/b1-78b9e19270a36df1.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0l5-4e7d0bd60ed3a66a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/c2z-702903d1c83e2892.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/f9-038758dcbfe08dab.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0k-b7df0a74912a42bf.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/sp-2bbb73a69162e23b.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/og-93ae8b358505d372.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/q5-fc1f2b54f1d10663.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/qod-79dedcb2e7d13687.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/du-dc47d4c63c11d935.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ql0-52233a028822e2ef.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/20-69395c63e3e4b526.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/dsj-4cca625a20b72565.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wp-2413e3416979956b.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/e84-4da7096eb6b55d6d.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0pz-9106948caa5e015a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7mh-23cfe0e4aeb0e064.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bsb-4f4ac8c20da73fac.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/46-90fb1c98afc2cd5a.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/jga-3f6576d6f86d4e43.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/me-dfecacc72ba91157.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0dh-eb608edac481e761.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xc-f5d51b571cd317cb.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7c-a3aa1fcc96ee2fd8.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4b-c63a3f3820a90d31.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/qv-5ae49538d02f5979.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/gtp-bb135af2e2127c94.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-view-bdd08a7c4b6a41ae.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.bad9067f7ec9f6d1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/ql0.98d0f45ff87777ec.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/46.3e560835284be460.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/xc.13e027137438fd9f.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/4b.e524e9f270f17ecc.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/qv.9631adba022ab320.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-view.0fa77f65e2dd93f8.module.css" />
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hu-0d4aa5ab34269698.js" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/31k-472f1465ccb55aa9.js" />
|
||
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/notifications-subscriptions-menu-bcafd20009b025c8.js" defer="defer"></script>
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.bad9067f7ec9f6d1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.2bf6718a44c2b857.module.css" />
|
||
|
||
|
||
<title>phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide · GitHub</title>
|
||
|
||
|
||
|
||
<meta name="route-pattern" content="/:user_id/:repository/blob/*name(/*path)" data-turbo-transient>
|
||
<meta name="route-controller" content="blob" data-turbo-transient>
|
||
<meta name="route-action" content="show" data-turbo-transient>
|
||
<meta name="fetch-nonce" content="v2:18b05885-5ddd-3d33-92fe-1a864cbad779">
|
||
|
||
|
||
<meta name="current-catalog-service-hash" content="f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb">
|
||
|
||
|
||
<meta name="request-id" content="E9A6:1E3C7A:17D115:2964B3:6AA4F766" data-pjax-transient="true"/><meta name="html-safe-nonce" content="d2fba2e4893d736deae803ef8955aa08f6461aa89c379a11977d610c911dba21" data-pjax-transient="true"/><meta name="visitor-payload" content="eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJFOUE2OjFFM0M3QToxN0QxMTU6Mjk2NEIzOjZBQTRGNzY2IiwidmlzaXRvcl9pZCI6Ijc3OTg2OTA1Mjg1MzY4ODUwOTQiLCJyZWdpb25fZWRnZSI6Indlc3R1czMiLCJyZWdpb25fcmVuZGVyIjoid2VzdHVzMyJ9" data-pjax-transient="true"/><meta name="visitor-hmac" content="b633cf0f60b7506f7f9c77e30b28d37b4dbd56472e2b10cd26ed53368f30a4f8" data-pjax-transient="true"/>
|
||
|
||
|
||
<meta name="hovercard-subject-tag" content="repository:1188278606" data-turbo-transient>
|
||
|
||
|
||
<meta name="github-keyboard-shortcuts" content="repository,source-code,file-tree,copilot" data-turbo-transient="true" />
|
||
|
||
|
||
<meta name="selected-link" value="repo_source" data-turbo-transient>
|
||
<link rel="assets" href="https://github.githubassets.com/">
|
||
|
||
<meta name="google-site-verification" content="Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I">
|
||
|
||
<meta name="octolytics-url" content="https://collector.github.com/github/collect" />
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="analytics-location" content="/<user-name>/<repo-name>/blob/show" data-turbo-transient="true" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="user-login" content="">
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width">
|
||
|
||
|
||
|
||
<meta name="description" content="Global maritime intelligence platform for real-time vessel tracking, AIS data analysis, sanctions monitoring, shipping routes, port activity, anomaly detection, OSINT analytics, and geospatial insights - phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide">
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
|
||
|
||
<link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
|
||
<meta property="fb:app_id" content="1401488693436528">
|
||
<meta name="apple-itunes-app" content="app-id=1477376905, app-argument=https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md" />
|
||
|
||
<meta name="twitter:image" content="https://opengraph.githubassets.com/912bbd2b3c1aef0cdcdc9fc15d47c6ecbe81fdc11a694ae6710848f7577b3c8f/tg12/phantomtide" /><meta name="twitter:site" content="@github" /><meta name="twitter:card" content="summary_large_image" /><meta name="twitter:title" content="phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide" /><meta name="twitter:description" content="Global maritime intelligence platform for real-time vessel tracking, AIS data analysis, sanctions monitoring, shipping routes, port activity, anomaly detection, OSINT analytics, and geospatial insi..." />
|
||
<meta property="og:image" content="https://opengraph.githubassets.com/912bbd2b3c1aef0cdcdc9fc15d47c6ecbe81fdc11a694ae6710848f7577b3c8f/tg12/phantomtide" /><meta property="og:image:alt" content="Global maritime intelligence platform for real-time vessel tracking, AIS data analysis, sanctions monitoring, shipping routes, port activity, anomaly detection, OSINT analytics, and geospatial insi..." /><meta property="og:image:width" content="1200" /><meta property="og:image:height" content="600" /><meta property="og:site_name" content="GitHub" /><meta property="og:type" content="object" /><meta property="og:title" content="phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide" /><meta property="og:url" content="https://github.com/tg12/phantomtide/blob/main/docs/oom-postmortem.md" /><meta property="og:description" content="Global maritime intelligence platform for real-time vessel tracking, AIS data analysis, sanctions monitoring, shipping routes, port activity, anomaly detection, OSINT analytics, and geospatial insi..." />
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="hostname" content="github.com">
|
||
|
||
|
||
|
||
<meta name="expected-hostname" content="github.com">
|
||
|
||
|
||
<meta http-equiv="x-pjax-version" content="37045a1f63d0a2ad20afa416c72b63d666b66fb371c3e7720d263eb92dda33a7" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-csp-version" content="f2311ccc2e94cef09d9c62c8c090301d08823a2fd66b86a913b7d968c86acb1a" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-css-version" content="b7a4653359baeee3d827ca91d7c149e4962234557341efae58fad4aa5ce82a0a" data-turbo-track="reload">
|
||
<meta http-equiv="x-pjax-js-version" content="96efb01f6430a1ad6e22f9e0b8a04625dd76a461e93da809464e8d1f9c36c59c" data-turbo-track="reload">
|
||
|
||
<meta name="turbo-cache-control" content="no-preview" data-turbo-transient="">
|
||
|
||
<meta name="turbo-cache-control" content="no-cache" data-turbo-transient>
|
||
|
||
<meta data-hydrostats="publish">
|
||
|
||
<meta name="go-import" content="github.com/tg12/phantomtide git https://github.com/tg12/phantomtide.git">
|
||
|
||
<meta name="octolytics-dimension-user_id" content="12201893" /><meta name="octolytics-dimension-user_login" content="tg12" /><meta name="octolytics-dimension-repository_id" content="1188278606" /><meta name="octolytics-dimension-repository_nwo" content="tg12/phantomtide" /><meta name="octolytics-dimension-repository_public" content="true" /><meta name="octolytics-dimension-repository_is_fork" content="false" /><meta name="octolytics-dimension-repository_network_root_id" content="1188278606" /><meta name="octolytics-dimension-repository_network_root_nwo" content="tg12/phantomtide" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="turbo-body-classes" content="logged-out env-production page-responsive">
|
||
<meta name="disable-turbo" content="false">
|
||
|
||
|
||
<meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
|
||
|
||
|
||
<meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
|
||
|
||
<meta name="release" content="ee0d1b245383d5b08b4f8b7ecd18cda5bdccbbe4" data-turbo-track="reload">
|
||
<meta name="ui-target" content="full">
|
||
|
||
<link rel="mask-icon" href="https://github.githubassets.com/assets/pinned-octocat-093da3e6fa40.svg" color="#000000">
|
||
<link rel="alternate icon" class="js-site-favicon" type="image/png" href="https://github.githubassets.com/favicons/favicon.png">
|
||
<link rel="icon" class="js-site-favicon" type="image/svg+xml" href="https://github.githubassets.com/favicons/favicon.svg" data-base-href="https://github.githubassets.com/favicons/favicon">
|
||
|
||
<meta name="theme-color" content="#1e2327">
|
||
<meta name="color-scheme" content="light dark" />
|
||
|
||
|
||
<link rel="manifest" href="/manifest.json" crossOrigin="use-credentials">
|
||
|
||
</head>
|
||
|
||
<body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
|
||
<div data-turbo-body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
|
||
<div id="__primerPortalRoot__" style="z-index: 1000; position: absolute; width: 100%;" data-turbo-permanent></div>
|
||
|
||
|
||
<div class="position-relative header-wrapper js-header-wrapper ">
|
||
<a href="#start-of-content" data-skip-target-assigned="false" class="px-2 tmp-py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content">Skip to content</a>
|
||
|
||
<span data-view-component="true" class="progress-pjax-loader Progress position-fixed width-full">
|
||
<span style="width: 0%;" data-view-component="true" class="Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis"></span>
|
||
</span>
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/l7j-a8c3c3ab9790e365.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-98c649c7176c489a.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.bad9067f7ec9f6d1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog.b5f61eada8ea05c6.module.css" />
|
||
|
||
<react-partial
|
||
partial-name="keyboard-shortcuts-dialog"
|
||
data-ssr="false"
|
||
data-attempted-ssr="false"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"docsUrl":"https://docs.github.com/get-started/accessibility/keyboard-shortcuts"}}</script>
|
||
<div data-target="react-partial.reactRoot"></div>
|
||
</react-partial>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nvz-deeea5270ef3dfb4.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/mz-a6c5397389be3fff.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fz5-23f28c0f388769c9.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/lht-b91be4a9ea8a3210.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/p1-76a75a6081059b19.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3d-82c347d20a13fb90.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/tua-81c88ff7e39af43c.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4s-854aef8a5dc4797a.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/marketing-header-b4f24b72e9c5634b.js" fetchpriority="low" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.bad9067f7ec9f6d1.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-brand-css.206bbd4fd232654e.module.css" />
|
||
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/4s.795b697dd2335e43.module.css" />
|
||
|
||
<react-partial
|
||
partial-name="marketing-header"
|
||
data-ssr="true"
|
||
data-attempted-ssr="true"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"color_mode":"dark","logged_in":false,"marketing_page":false,"home_path":"/","login_path":"/login?return_to=https%3A%2F%2Fgithub.com%2Ftg12%2Fphantomtide%2Fblob%2Fmain%2Fdocs%252Foom-postmortem.md","signup_path":"/signup?ref_cta=Sign+up\u0026ref_loc=header+logged+out\u0026ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow\u0026source=header-repo\u0026source_repo=tg12%2Fphantomtide","signup_enabled":true,"is_signup_controller":false,"show_search_and_nav":true,"hide_search":false,"private_mode_enabled":false,"should_use_dotcom_links":true,"auth_hydro_click":"{\"event_type\":\"authentication.click\",\"payload\":{\"location_in_page\":\"site header menu\",\"repository_id\":null,\"auth_type\":\"SIGN_UP\",\"originating_url\":\"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md\",\"user_id\":null}}","auth_hydro_click_hmac":"9d740f7e310ea6141b17ee6aeb29a851c429bba05545ba20bef12ffaf11d7049","overlay":false,"fixed":false}}</script>
|
||
<div data-target="react-partial.reactRoot"><div data-color-mode="dark" data-light-theme="light" data-dark-theme="dark"><header class="MarketingHeader-module__root__Tk7n3 HeaderMktg header-logged-out" role="banner" data-marketing-header="true" data-color-mode="dark" data-light-theme="light" data-dark-theme="dark" data-is-top="true"><h2 class="MarketingHeader-module__visuallyHidden__sqKsl">Navigation Menu</h2><button type="button" class="MarketingHeader-module__backdrop__sw4RU" aria-label="Close navigation menu"></button><div class="MarketingHeader-module__bar__mBSyE"><div class="MarketingHeader-module__topRow__yeury"><div class="MarketingHeader-module__toggleSlot__hDxbh"><button type="button" class="HeaderMenuToggle-module__toggle__i8EiC" aria-label="Toggle navigation" aria-expanded="false"><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span></button></div><a href="/" aria-label="Homepage" class="HeaderLogo-module__logo__UFyHI" data-analytics-event="{"action":"homepage","tag":"link","context":"logo","location":"header","label":"homepage_link_logo_header"}"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github" viewBox="0 0 24 24" width="32" height="32" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path></svg></a><div class="AuthCTAs-module__mobileActions__NNzeV"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/login?return_to=https%3A%2F%2Fgithub.com%2Ftg12%2Fphantomtide%2Fblob%2Fmain%2Fdocs%252Foom-postmortem.md" data-analytics-event="{"action":"sign_in","tag":"link","context":"auth_cta","location":"header","label":"sign_in_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="9d740f7e310ea6141b17ee6aeb29a851c429bba05545ba20bef12ffaf11d7049"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_3dd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_3dd_">Appearance settings</div></div></div><div class="MarketingHeader-module__menu__GIy3y"><div class="MarketingHeader-module__menuWrapper__owstH"><nav class="MarketingNavigation-module__nav__W0KYY" aria-label="Global"><ul class="MarketingNavigation-module__list__tFbMb"><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_nd_">Platform<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5knd_">AI CODE CREATION</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot" data-analytics-event="{"action":"github_copilot","tag":"link","context":"platform","location":"navbar","label":"github_copilot_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>GitHub Copilot</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Write better code with AI</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/ai/github-app" data-analytics-event="{"action":"github_copilot_app","tag":"link","context":"platform","location":"navbar","label":"github_copilot_app_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.766 11.328c-2.063-.25-3.516-1.734-3.516-3.656 0-.781.281-1.625.75-2.188-.203-.515-.172-1.609.063-2.062.625-.078 1.468.25 1.968.703.594-.187 1.219-.281 1.985-.281.765 0 1.39.094 1.953.265.484-.437 1.344-.765 1.969-.687.218.422.25 1.515.046 2.047.5.593.766 1.39.766 2.203 0 1.922-1.453 3.375-3.547 3.64.531.344.89 1.094.89 1.954v1.625c0 .468.391.734.86.547C13.781 14.359 16 11.53 16 8.03 16 3.61 12.406 0 7.984 0 3.563 0 0 3.61 0 8.031a7.88 7.88 0 0 0 5.172 7.422c.422.156.828-.125.828-.547v-1.25c-.219.094-.5.156-.75.156-1.031 0-1.64-.562-2.078-1.609-.172-.422-.36-.672-.719-.719-.187-.015-.25-.093-.25-.187 0-.188.313-.328.625-.328.453 0 .844.281 1.25.86.313.452.64.655 1.031.655s.641-.14 1-.5c.266-.265.47-.5.657-.656"></path></svg>GitHub Copilot app</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Direct agents from issue to merge</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/mcp" data-analytics-event="{"action":"mcp_registry","tag":"link","context":"platform","location":"navbar","label":"mcp_registry_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mcp NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.52 1.12a3.578 3.578 0 0 1 6.078 2.98 3.578 3.578 0 0 1 2.982 6.08l-3.292 3.293a.252.252 0 0 0 0 .354l.843.843a.749.749 0 1 1-1.06 1.06l-.844-.843a1.75 1.75 0 0 1 0-2.474L13.52 9.12a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0L7.731 9.03A.75.75 0 0 1 6.67 7.97l2.85-2.85a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0l-4.799 4.8A.75.75 0 0 1 .72 5.92Z"></path><path d="M7.52 3.12a.749.749 0 1 1 1.06 1.06L5.731 7.03A2.079 2.079 0 0 0 8.67 9.97l2.85-2.85a.749.749 0 1 1 1.06 1.06l-2.849 2.85A3.578 3.578 0 0 1 4.67 5.97Z"></path></svg>MCP Registry</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Integrate external tools</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9knd_">DEVELOPER WORKFLOWS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/actions" data-analytics-event="{"action":"actions","tag":"link","context":"platform","location":"navbar","label":"actions_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-workflow NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path></svg>Actions</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Automate any workflow</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/codespaces" data-analytics-event="{"action":"codespaces","tag":"link","context":"platform","location":"navbar","label":"codespaces_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codespaces NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 11.25c0-.966.784-1.75 1.75-1.75h12.5c.966 0 1.75.784 1.75 1.75v3A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm2-9.5C2 .784 2.784 0 3.75 0h8.5C13.216 0 14 .784 14 1.75v5a1.75 1.75 0 0 1-1.75 1.75h-8.5A1.75 1.75 0 0 1 2 6.75Zm1.75-.25a.25.25 0 0 0-.25.25v5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5a.25.25 0 0 0-.25-.25Zm-2 9.5a.25.25 0 0 0-.25.25v3c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-3a.25.25 0 0 0-.25-.25Z"></path><path d="M7 12.75a.75.75 0 0 1 .75-.75h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path></svg>Codespaces</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Instant dev environments</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/issues" data-analytics-event="{"action":"issues","tag":"link","context":"platform","location":"navbar","label":"issues_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-issue-opened NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path></svg>Issues</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Plan and track work</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-review" data-analytics-event="{"action":"code_review","tag":"link","context":"platform","location":"navbar","label":"code_review_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path></svg>Code Review</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Manage code changes</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-quality" data-analytics-event="{"action":"code_quality","tag":"link","context":"platform","location":"navbar","label":"code_quality_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codescan-checkmark NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.28 6.28a.75.75 0 1 0-1.06-1.06L6.25 8.19l-.97-.97a.75.75 0 0 0-1.06 1.06l1.5 1.5a.75.75 0 0 0 1.06 0l3.5-3.5Z"></path><path d="M7.5 15a7.5 7.5 0 1 1 5.807-2.754l2.473 2.474a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2.474-2.473A7.472 7.472 0 0 1 7.5 15Zm0-13.5a6 6 0 1 0 4.094 10.386.748.748 0 0 1 .293-.292 6.002 6.002 0 0 0 1.117-6.486A6.002 6.002 0 0 0 7.5 1.5Z"></path></svg>Code Quality</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enforce quality at merge</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dknd_">APPLICATION SECURITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{"action":"github_advanced_security","tag":"link","context":"platform","location":"navbar","label":"github_advanced_security_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Find and fix vulnerabilities</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/code-security" data-analytics-event="{"action":"code_security","tag":"link","context":"platform","location":"navbar","label":"code_security_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code-square NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>Code security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Secure your code as you build</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/secret-protection" data-analytics-event="{"action":"secret_protection","tag":"link","context":"platform","location":"navbar","label":"secret_protection_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-lock NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M4 4a4 4 0 0 1 8 0v2h.25c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 12.25 15h-8.5A1.75 1.75 0 0 1 2 13.25v-5.5C2 6.784 2.784 6 3.75 6H4Zm8.25 3.5h-8.5a.25.25 0 0 0-.25.25v5.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25ZM10.5 6V4a2.5 2.5 0 1 0-5 0v2Z"></path></svg>Secret protection</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Stop leaks before they start</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ NavGroup-module__hasSeparator__FnMrN"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_hknd_">EXPLORE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_hknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/why-github" data-analytics-event="{"action":"why_github","tag":"link","context":"platform","location":"navbar","label":"why_github_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Why GitHub</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{"action":"documentation","tag":"link","context":"platform","location":"navbar","label":"documentation_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog" data-analytics-event="{"action":"blog","tag":"link","context":"platform","location":"navbar","label":"blog_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Blog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog/changelog" data-analytics-event="{"action":"changelog","tag":"link","context":"platform","location":"navbar","label":"changelog_link_platform_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Changelog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/marketplace" data-analytics-event="{"action":"marketplace","tag":"link","context":"platform","location":"navbar","label":"marketplace_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Marketplace</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/features" data-analytics-event="{"action":"view_all_features","tag":"link","context":"platform","location":"navbar","label":"view_all_features_link_platform_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all features</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_17d_">Solutions<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_17d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5l7d_">BY COMPANY SIZE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise" data-analytics-event="{"action":"enterprises","tag":"link","context":"solutions","location":"navbar","label":"enterprises_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Enterprises</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/team" data-analytics-event="{"action":"small_and_medium_teams","tag":"link","context":"solutions","location":"navbar","label":"small_and_medium_teams_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Small and medium teams</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise/startups" data-analytics-event="{"action":"startups","tag":"link","context":"solutions","location":"navbar","label":"startups_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Startups</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/nonprofits" data-analytics-event="{"action":"nonprofits","tag":"link","context":"solutions","location":"navbar","label":"nonprofits_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Nonprofits</span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9l7d_">BY USE CASE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/app-modernization" data-analytics-event="{"action":"app_modernization","tag":"link","context":"solutions","location":"navbar","label":"app_modernization_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">App Modernization</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devsecops" data-analytics-event="{"action":"devsecops","tag":"link","context":"solutions","location":"navbar","label":"devsecops_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevSecOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devops" data-analytics-event="{"action":"devops","tag":"link","context":"solutions","location":"navbar","label":"devops_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/ci-cd" data-analytics-event="{"action":"ci/cd","tag":"link","context":"solutions","location":"navbar","label":"ci/cd_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">CI/CD</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/use-case" data-analytics-event="{"action":"view_all_use_cases","tag":"link","context":"solutions","location":"navbar","label":"view_all_use_cases_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all use cases</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dl7d_">BY INDUSTRY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dl7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/healthcare" data-analytics-event="{"action":"healthcare","tag":"link","context":"solutions","location":"navbar","label":"healthcare_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Healthcare</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/financial-services" data-analytics-event="{"action":"financial_services","tag":"link","context":"solutions","location":"navbar","label":"financial_services_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Financial services</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/manufacturing" data-analytics-event="{"action":"manufacturing","tag":"link","context":"solutions","location":"navbar","label":"manufacturing_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Manufacturing</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/government" data-analytics-event="{"action":"government","tag":"link","context":"solutions","location":"navbar","label":"government_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Government</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/industry" data-analytics-event="{"action":"view_all_industries","tag":"link","context":"solutions","location":"navbar","label":"view_all_industries_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all industries</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions" data-analytics-event="{"action":"view_all_solutions","tag":"link","context":"solutions","location":"navbar","label":"view_all_solutions_link_solutions_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all solutions</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_1nd_">Resources<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_1nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5lnd_">EXPLORE BY TOPIC</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=ai" data-analytics-event="{"action":"ai","tag":"link","context":"resources","location":"navbar","label":"ai_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">AI</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=software-development" data-analytics-event="{"action":"software_development","tag":"link","context":"resources","location":"navbar","label":"software_development_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Software Development</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=devops" data-analytics-event="{"action":"devops","tag":"link","context":"resources","location":"navbar","label":"devops_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=security" data-analytics-event="{"action":"security","tag":"link","context":"resources","location":"navbar","label":"security_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources/articles" data-analytics-event="{"action":"view_all_topics","tag":"link","context":"resources","location":"navbar","label":"view_all_topics_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all topics</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9lnd_">EXPLORE BY TYPE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/customer-stories" data-analytics-event="{"action":"customer_stories","tag":"link","context":"resources","location":"navbar","label":"customer_stories_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer stories</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/events" data-analytics-event="{"action":"events__webinars","tag":"link","context":"resources","location":"navbar","label":"events__webinars_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Events & webinars</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/whitepapers" data-analytics-event="{"action":"ebooks__reports","tag":"link","context":"resources","location":"navbar","label":"ebooks__reports_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Ebooks & reports</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/executive-insights" data-analytics-event="{"action":"business_insights","tag":"link","context":"resources","location":"navbar","label":"business_insights_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Business insights</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://skills.github.com" data-analytics-event="{"action":"github_skills","tag":"link","context":"resources","location":"navbar","label":"github_skills_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Skills</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dlnd_">SUPPORT & SERVICES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dlnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{"action":"documentation","tag":"link","context":"resources","location":"navbar","label":"documentation_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://support.github.com" data-analytics-event="{"action":"customer_support","tag":"link","context":"resources","location":"navbar","label":"customer_support_link_resources_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer support</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/orgs/community/discussions" data-analytics-event="{"action":"community_forum","tag":"link","context":"resources","location":"navbar","label":"community_forum_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Community forum</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trust-center" data-analytics-event="{"action":"trust_center","tag":"link","context":"resources","location":"navbar","label":"trust_center_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trust center</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/partners" data-analytics-event="{"action":"partners","tag":"link","context":"resources","location":"navbar","label":"partners_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Partners</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources" data-analytics-event="{"action":"view_all_resources","tag":"link","context":"resources","location":"navbar","label":"view_all_resources_link_resources_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all resources</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_27d_">Open Source<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_27d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5m7d_">COMMUNITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/open-source/sponsors" data-analytics-event="{"action":"github_sponsors","tag":"link","context":"open_source","location":"navbar","label":"github_sponsors_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sponsor-tiers NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.586 1C12.268 1 13.5 2.37 13.5 4.25c0 1.745-.996 3.359-2.622 4.831-.166.15-.336.297-.509.438l1.116 5.584a.75.75 0 0 1-.991.852l-2.409-.876a.25.25 0 0 0-.17 0l-2.409.876a.75.75 0 0 1-.991-.852L5.63 9.519a13.78 13.78 0 0 1-.51-.438C3.497 7.609 2.5 5.995 2.5 4.25 2.5 2.37 3.732 1 5.414 1c.963 0 1.843.403 2.474 1.073L8 2.198l.112-.125a3.385 3.385 0 0 1 2.283-1.068L10.586 1Zm-3.621 9.495-.718 3.594 1.155-.42a1.75 1.75 0 0 1 1.028-.051l.168.051 1.154.42-.718-3.592c-.199.13-.37.235-.505.314l-.169.097a.75.75 0 0 1-.72 0 9.54 9.54 0 0 1-.515-.308l-.16-.105ZM10.586 2.5c-.863 0-1.611.58-1.866 1.459-.209.721-1.231.721-1.44 0C7.025 3.08 6.277 2.5 5.414 2.5 4.598 2.5 4 3.165 4 4.25c0 1.23.786 2.504 2.128 3.719.49.443 1.018.846 1.546 1.198l.325.21.076-.047.251-.163a13.341 13.341 0 0 0 1.546-1.198C11.214 6.754 12 5.479 12 4.25c0-1.085-.598-1.75-1.414-1.75Z"></path></svg>GitHub Sponsors</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Fund open source developers</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9m7d_">PROGRAMS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://securitylab.github.com" data-analytics-event="{"action":"security_lab","tag":"link","context":"open_source","location":"navbar","label":"security_lab_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security Lab</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://maintainers.github.com" data-analytics-event="{"action":"maintainer_community","tag":"link","context":"open_source","location":"navbar","label":"maintainer_community_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Maintainer Community</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://stars.github.com" data-analytics-event="{"action":"github_stars","tag":"link","context":"open_source","location":"navbar","label":"github_stars_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Stars</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://archiveprogram.github.com" data-analytics-event="{"action":"archive_program","tag":"link","context":"open_source","location":"navbar","label":"archive_program_link_open_source_navbar"}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Archive Program</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dm7d_">REPOSITORIES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dm7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/topics" data-analytics-event="{"action":"topics","tag":"link","context":"open_source","location":"navbar","label":"topics_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Topics</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trending" data-analytics-event="{"action":"trending","tag":"link","context":"open_source","location":"navbar","label":"trending_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trending</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/collections" data-analytics-event="{"action":"collections","tag":"link","context":"open_source","location":"navbar","label":"collections_link_open_source_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Collections</span></a></li></ul></div></li></ul></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_2nd_">Enterprise<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_2nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5mnd_">ENTERPRISE SOLUTIONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise" data-analytics-event="{"action":"enterprise_platform","tag":"link","context":"enterprise","location":"navbar","label":"enterprise_platform_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-stack NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.122.392a1.75 1.75 0 0 1 1.756 0l5.003 2.902c.83.481.83 1.68 0 2.162L8.878 8.358a1.75 1.75 0 0 1-1.756 0L2.119 5.456a1.251 1.251 0 0 1 0-2.162ZM8.125 1.69a.248.248 0 0 0-.25 0l-4.63 2.685 4.63 2.685a.248.248 0 0 0 .25 0l4.63-2.685ZM1.601 7.789a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0L1.874 8.814A.75.75 0 0 1 1.6 7.789Zm0 3.5a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0l-5.248-3.044a.75.75 0 0 1-.273-1.025Z"></path></svg>Enterprise platform</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">AI-powered developer platform</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9mnd_">AVAILABLE ADD-ONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{"action":"github_advanced_security","tag":"link","context":"enterprise","location":"navbar","label":"github_advanced_security_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade security features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot/copilot-business" data-analytics-event="{"action":"copilot_for_business","tag":"link","context":"enterprise","location":"navbar","label":"copilot_for_business_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>Copilot for Business</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade AI features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise/premium-support" data-analytics-event="{"action":"premium_support","tag":"link","context":"enterprise","location":"navbar","label":"premium_support_link_enterprise_navbar"}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-comment-discussion NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1h8.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 10.25 10H7.061l-2.574 2.573A1.458 1.458 0 0 1 2 11.543V10h-.25A1.75 1.75 0 0 1 0 8.25v-5.5C0 1.784.784 1 1.75 1ZM1.5 2.75v5.5c0 .138.112.25.25.25h1a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h3.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13 2a.25.25 0 0 0-.25-.25h-.5a.75.75 0 0 1 0-1.5h.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 14.25 12H14v1.543a1.458 1.458 0 0 1-2.487 1.03L9.22 12.28a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l2.22 2.22v-2.19a.75.75 0 0 1 .75-.75h1a.25.25 0 0 0 .25-.25Z"></path></svg>Premium Support</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade 24/7 support</span></span></a></li></ul></div></li></ul></div></div></li><li><a href="https://github.com/pricing" data-analytics-event="{"action":"pricing","tag":"link","context":"pricing","location":"navbar","label":"pricing_link_pricing_navbar"}" class="MarketingNavigation-module__navLink__hUomM">Pricing</a></li></ul></nav><div class="MarketingHeader-module__ctaContainer__tBmPz"><div class="HeaderSearch-module__searchSlot__oVOUS"><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderSearch-module__trigger__zsF9q" type="button" aria-haspopup="dialog" aria-expanded="false" aria-label="Search or jump to, type / to search" data-analytics-event="{"action":"searchbar","tag":"input","context":"global","location":"navbar","label":"searchbar_input_global_navbar"}"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"><span class="HeaderSearch-module__content__kMpxU"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search HeaderSearch-module__icon__wcrHX" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg><span class="HeaderSearch-module__label__d1iWG">Search</span><kbd class="HeaderSearch-module__kbd__HNG0o" aria-hidden="true">/</kbd></span></span></span></button><div class="d-none"></div></div><div class="AuthCTAs-module__signInWrap__q2P60"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq AuthCTAs-module__desktopActionGap__UZuXT AuthCTAs-module__hiddenBelowLg__BfKBw" href="/login?return_to=https%3A%2F%2Fgithub.com%2Ftg12%2Fphantomtide%2Fblob%2Fmain%2Fdocs%252Foom-postmortem.md" data-analytics-event="{"action":"sign_in","tag":"link","context":"auth_cta","location":"header","label":"sign_in_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="9d740f7e310ea6141b17ee6aeb29a851c429bba05545ba20bef12ffaf11d7049"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a></div><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--secondary___gHnw_ Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow&source=header-repo&source_repo=tg12%2Fphantomtide" data-analytics-event="{"action":"sign_up","tag":"link","context":"auth_cta","location":"header","label":"sign_up_link_auth_cta_header"}" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="9d740f7e310ea6141b17ee6aeb29a851c429bba05545ba20bef12ffaf11d7049"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-secondary___eJ0_a">Sign up</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_fbd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_fbd_">Appearance settings</div></div></div></div></div><div class="MarketingHeader-module__bottomBorder__uZT38" aria-hidden="true"></div></header></div></div>
|
||
</react-partial>
|
||
|
||
|
||
|
||
<div hidden="hidden" data-view-component="true" class="js-stale-session-flash stale-session-flash flash flash-warn flash-full">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
|
||
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<span class="js-stale-session-flash-signed-in" hidden>You signed in with another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
<span class="js-stale-session-flash-signed-out" hidden>You signed out in another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
<span class="js-stale-session-flash-switched" hidden>You switched accounts on another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
|
||
|
||
<button id="icon-button-958c71d6-96c9-46d3-bdc2-6a4c10c75a78" aria-labelledby="tooltip-5b0d4a6a-10e8-49ea-bbe6-4eedb996136a" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium flash-close js-flash-close"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x Button-visual">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button><tool-tip id="tooltip-5b0d4a6a-10e8-49ea-bbe6-4eedb996136a" for="icon-button-958c71d6-96c9-46d3-bdc2-6a4c10c75a78" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Dismiss alert</tool-tip>
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="start-of-content" class="show-on-focus"></div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="js-flash-container" class="flash-container" data-turbo-replace>
|
||
|
||
|
||
|
||
|
||
|
||
<template class="js-flash-template">
|
||
|
||
<div class="flash flash-full {{ className }}">
|
||
<div >
|
||
<button autofocus class="flash-close js-flash-close" type="button" aria-label="Dismiss this message">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
<div aria-atomic="true" role="alert" class="js-flash-alert">
|
||
|
||
<div>{{ message }}</div>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</template>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div
|
||
class="application-main "
|
||
data-commit-hovercards-enabled
|
||
data-discussion-hovercards-enabled
|
||
data-issue-and-pr-hovercards-enabled
|
||
data-project-hovercards-enabled
|
||
>
|
||
<div itemscope itemtype="http://schema.org/SoftwareSourceCode" class="">
|
||
<main id="js-repo-pjax-container" >
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="repository-container-header" class="tmp-pt-3 hide-full-screen" style="background-color: var(--page-header-bgColor, var(--color-page-header-bg));" data-turbo-replace>
|
||
|
||
<div class="d-flex flex-nowrap flex-justify-end tmp-mb-3 tmp-px-3 tmp-px-lg-5" style="gap: 1rem;">
|
||
|
||
<div class="flex-auto min-width-0 width-fit">
|
||
|
||
<div class=" d-flex flex-wrap flex-items-center wb-break-word f3 text-normal">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo color-fg-muted mr-2 tmp-mr-2">
|
||
<path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path>
|
||
</svg>
|
||
|
||
<span class="author flex-self-stretch" itemprop="author">
|
||
<a class="url fn" rel="author" data-hovercard-type="user" data-hovercard-url="/users/tg12/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="/tg12">
|
||
tg12
|
||
</a> </span>
|
||
<span class="mx-1 flex-self-stretch color-fg-muted">/</span>
|
||
<strong itemprop="name" class="mr-2 flex-self-stretch">
|
||
<a data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" href="/tg12/phantomtide">phantomtide</a>
|
||
</strong>
|
||
|
||
<span></span><span class="Label Label--secondary v-align-middle mr-1">Public</span>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div id="repository-details-container" class="flex-shrink-0" data-turbo-replace style="max-width: 70%;">
|
||
<ul class="pagehead-actions flex-shrink-0 d-none d-md-inline" style="padding: 2px 0;">
|
||
|
||
|
||
|
||
<li>
|
||
<a href="/login?return_to=%2Ftg12%2Fphantomtide" rel="nofollow" id="repository-details-watch-button" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"notification subscription menu watch","repository_id":null,"auth_type":"LOG_IN","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="647fc2b57e5583be687d3dc9390f39b6dc96cd00dd4a91792ac9cff47b38f483" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2 tmp-mr-2">
|
||
<path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path>
|
||
</svg>Notifications
|
||
</a> <tool-tip id="tooltip-a5547ca6-111e-4f14-9767-b35988361548" for="repository-details-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute">You must be signed in to change notification settings</tool-tip>
|
||
|
||
</li>
|
||
|
||
<li>
|
||
<a icon="repo-forked" id="fork-button" href="/login?return_to=%2Ftg12%2Fphantomtide" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"repo details fork button","repository_id":1188278606,"auth_type":"LOG_IN","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="4600ba1fa63484976fc9fd2ed6e596c3e7d8a129ec8dac24d168385be8445c61" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2 tmp-mr-2">
|
||
<path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
|
||
</svg>Fork
|
||
<span id="repo-network-counter" data-pjax-replace="true" data-turbo-replace="true" title="18" data-view-component="true" class="Counter">18</span>
|
||
</a>
|
||
</li>
|
||
|
||
<li>
|
||
<div data-view-component="true" class="BtnGroup d-flex">
|
||
<a href="/login?return_to=%2Ftg12%2Fphantomtide" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"star button","repository_id":1188278606,"auth_type":"LOG_IN","originating_url":"https://github.com/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","user_id":null}}" data-hydro-click-hmac="0eab208b5fad8ed585f0dfe1cef1573de9b657d01b8037c0e0d0c24293e8cba8" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2 tmp-mr-2">
|
||
<path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
|
||
</svg><span data-view-component="true" class="d-inline">
|
||
Star
|
||
</span> <span id="repo-stars-counter-star" aria-label="123 users starred this repository" data-singular-suffix="user starred this repository" data-plural-suffix="users starred this repository" data-turbo-replace="true" title="123" data-view-component="true" class="Counter js-social-count">123</span>
|
||
</a></div>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="responsive-meta-container" data-turbo-replace>
|
||
</div>
|
||
|
||
|
||
<nav data-pjax="#js-repo-pjax-container" aria-label="Repository" data-view-component="true" class="js-repo-nav js-sidenav-container-pjax js-responsive-underlinenav overflow-hidden UnderlineNav px-3 tmp-px-3 px-md-4 tmp-px-md-4 px-lg-5 tmp-px-lg-5">
|
||
|
||
<ul data-view-component="true" class="UnderlineNav-body list-style-none">
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="code-tab" href="/tg12/phantomtide" data-tab-item="i0code-tab" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches repo_packages repo_deployments repo_attestations /tg12/phantomtide" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g c" data-command-id="repositories:go-to-code" data-react-nav="code-view" data-react-nav-anchor="code-view-repo-link" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Code","target":"UNDERLINE_NAV.TAB"}" aria-current="page" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item selected">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
<span data-content="Code">Code</span>
|
||
<span id="code-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="issues-tab" href="/tg12/phantomtide/issues" data-tab-item="i1issues-tab" data-selected-links="repo_issues repo_labels repo_milestones /tg12/phantomtide/issues" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g i" data-command-id="repositories:go-to-issues" data-react-nav="issues-react" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Issues","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
|
||
</svg>
|
||
<span data-content="Issues">Issues</span>
|
||
<span id="issues-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="4" data-view-component="true" class="Counter">4</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="pull-requests-tab" href="/tg12/phantomtide/pulls" data-tab-item="i2pull-requests-tab" data-selected-links="repo_pulls checks /tg12/phantomtide/pulls" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g p" data-command-id="repositories:go-to-pull-requests" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Pull requests","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
|
||
</svg>
|
||
<span data-content="Pull requests">Pull requests</span>
|
||
<span id="pull-requests-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="actions-tab" href="/tg12/phantomtide/actions" data-tab-item="i3actions-tab" data-selected-links="repo_actions /tg12/phantomtide/actions" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g a" data-command-id="repositories:go-to-actions" data-react-nav="actions-workflows" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Actions","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
|
||
</svg>
|
||
<span data-content="Actions">Actions</span>
|
||
<span id="actions-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="projects-tab" href="/tg12/phantomtide/projects" data-tab-item="i4projects-tab" data-selected-links="repo_projects new_repo_project repo_project /tg12/phantomtide/projects" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g b" data-command-id="repositories:go-to-projects" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Projects","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
|
||
</svg>
|
||
<span data-content="Projects">Projects</span>
|
||
<span id="projects-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="security-and-quality-tab" href="/tg12/phantomtide/security" data-tab-item="i5security-and-quality-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /tg12/phantomtide/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-command-id="repositories:go-to-security" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Security and quality","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<span data-content="Security and quality">Security and quality</span>
|
||
<span id="security-and-quality-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
|
||
|
||
|
||
|
||
</a></li>
|
||
<li data-view-component="true" class="d-inline-flex">
|
||
<a id="insights-tab" href="/tg12/phantomtide/pulse" data-tab-item="i6insights-tab" data-selected-links="repo_graphs repo_contributors dependency_graph dependabot_updates pulse people community /tg12/phantomtide/pulse" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-command-id="repositories:go-to-insights" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Insights","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
|
||
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph UnderlineNav-octicon d-none d-sm-inline">
|
||
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
|
||
</svg>
|
||
<span data-content="Insights">Insights</span>
|
||
<span id="insights-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
|
||
|
||
|
||
|
||
</a></li>
|
||
</ul>
|
||
<div style="visibility:hidden;" data-view-component="true" class="UnderlineNav-actions js-responsive-underlinenav-overflow position-absolute pr-3 tmp-pr-3 pr-md-4 tmp-pr-md-4 pr-lg-5 tmp-pr-lg-5 right-0"> <action-menu data-select-variant="none" data-view-component="true">
|
||
<focus-group direction="vertical" mnemonics retain>
|
||
<button id="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-button" popovertarget="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-overlay" aria-controls="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-list" aria-haspopup="true" aria-labelledby="tooltip-7bbe466d-23c6-41aa-926e-69ae34cf41b8" type="button" data-view-component="true" class="Button Button--iconOnly Button--secondary Button--medium UnderlineNav-item"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-kebab-horizontal Button-visual">
|
||
<path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path>
|
||
</svg>
|
||
</button><tool-tip id="tooltip-7bbe466d-23c6-41aa-926e-69ae34cf41b8" for="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-button" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Additional navigation options</tool-tip>
|
||
|
||
|
||
<anchored-position data-target="action-menu.overlay" id="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-overlay" anchor="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-button" align="start" side="outside-bottom" anchor-offset="normal" popover="auto" data-view-component="true">
|
||
<div data-view-component="true" class="Overlay Overlay--size-auto">
|
||
|
||
<div data-view-component="true" class="Overlay-body Overlay-body--paddingNone"> <action-list>
|
||
<div data-view-component="true">
|
||
<ul aria-labelledby="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-button" id="action-menu-18aa0fd0-8c64-4c63-ab49-a63e5d7c3f5d-list" role="menu" data-view-component="true" class="ActionListWrap--inset ActionListWrap">
|
||
<li hidden="hidden" data-menu-item="i0code-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-59ed76e2-a45c-460a-8956-16bb03ff1795" href="/tg12/phantomtide" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code">
|
||
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Code
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i1issues-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-cdecdd8c-bf90-4621-a4ea-544a465d618e" href="/tg12/phantomtide/issues" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened">
|
||
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Issues
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i2pull-requests-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-df1a5888-4d71-44fb-98f8-a229eaa2dc2f" href="/tg12/phantomtide/pulls" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request">
|
||
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Pull requests
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i3actions-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-f1d0e1d4-32fc-4444-88ed-c6dd39f2c09a" href="/tg12/phantomtide/actions" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play">
|
||
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Actions
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i4projects-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-8a5e2256-a073-4d98-97b3-9dd822af9c87" href="/tg12/phantomtide/projects" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table">
|
||
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Projects
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i5security-and-quality-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-3d94c72a-1149-4a77-993b-74ba014c0f84" href="/tg12/phantomtide/security" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield">
|
||
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Security and quality
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
<li hidden="hidden" data-menu-item="i6insights-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
|
||
|
||
|
||
<a tabindex="-1" id="item-a572bb9b-6214-4f85-b1b6-005c1bceaceb" href="/tg12/phantomtide/pulse" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
|
||
<span class="ActionListItem-visual ActionListItem-visual--leading">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph">
|
||
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
|
||
</svg>
|
||
</span>
|
||
|
||
<span data-view-component="true" class="ActionListItem-label">
|
||
Insights
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
</ul>
|
||
</div></action-list>
|
||
|
||
|
||
</div>
|
||
|
||
</div></anchored-position> </focus-group>
|
||
</action-menu></div>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<turbo-frame id="repo-content-turbo-frame" target="_top" data-turbo-action="advance" class="">
|
||
<div id="repo-content-pjax-container" class="repository-content " >
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<react-app
|
||
app-name="code-view"
|
||
initial-path="/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md"
|
||
style="display: block; min-height: calc(100vh - 64px);"
|
||
data-attempted-ssr="true"
|
||
data-ssr="true"
|
||
data-lazy="false"
|
||
data-alternate="false"
|
||
data-data-router-enabled="true"
|
||
data-react-profiling="false"
|
||
>
|
||
|
||
<script type="application/json" data-target="react-app.embeddedData">{"payload":{"codeViewBlobRoute":{"csv":null,"csvError":null,"headerInfo":{"toc":[{"level":1,"text":"How I Dragged Phantom Tide Out of an OOM Kill Loop","anchor":"how-i-dragged-phantom-tide-out-of-an-oom-kill-loop","htmlText":"How I Dragged Phantom Tide Out of an OOM Kill Loop"},{"level":2,"text":"I thought I was fixing a crash loop. I was really exposing the architecture.","anchor":"i-thought-i-was-fixing-a-crash-loop-i-was-really-exposing-the-architecture","htmlText":"I thought I was fixing a crash loop. I was really exposing the architecture."},{"level":2,"text":"The first problem: I was treating a stateful process like a stateless web app","anchor":"the-first-problem-i-was-treating-a-stateful-process-like-a-stateless-web-app","htmlText":"The first problem: I was treating a stateful process like a stateless web app"},{"level":2,"text":"The second problem: 28 MB in Redis became roughly 2 GB in Python","anchor":"the-second-problem-28-mb-in-redis-became-roughly-2-gb-in-python","htmlText":"The second problem: 28 MB in Redis became roughly 2 GB in Python"},{"level":2,"text":"The third problem: the scheduler was launching a coordinated attack on startup","anchor":"the-third-problem-the-scheduler-was-launching-a-coordinated-attack-on-startup","htmlText":"The third problem: the scheduler was launching a coordinated attack on startup"},{"level":2,"text":"The fourth problem: the fast loop was still colliding with the slow startup window","anchor":"the-fourth-problem-the-fast-loop-was-still-colliding-with-the-slow-startup-window","htmlText":"The fourth problem: the fast loop was still colliding with the slow startup window"},{"level":2,"text":"The fifth problem: ClickHouse was configured like it owned the box","anchor":"the-fifth-problem-clickhouse-was-configured-like-it-owned-the-box","htmlText":"The fifth problem: ClickHouse was configured like it owned the box"},{"level":2,"text":"The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse","anchor":"the-sixth-problem-huge-flushes-were-creating-secondary-memory-spikes-in-clickhouse","htmlText":"The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse"},{"level":2,"text":"The seventh problem: zero swap headroom made every spike fatal","anchor":"the-seventh-problem-zero-swap-headroom-made-every-spike-fatal","htmlText":"The seventh problem: zero swap headroom made every spike fatal"},{"level":2,"text":"What changed after the fixes","anchor":"what-changed-after-the-fixes","htmlText":"What changed after the fixes"},{"level":2,"text":"What this incident proved about the product","anchor":"what-this-incident-proved-about-the-product","htmlText":"What this incident proved about the product"},{"level":2,"text":"The uncomfortable truth","anchor":"the-uncomfortable-truth","htmlText":"The uncomfortable truth"},{"level":2,"text":"What still remains","anchor":"what-still-remains","htmlText":"What still remains"},{"level":2,"text":"The real lesson","anchor":"the-real-lesson","htmlText":"The real lesson"}]},"issueTemplate":null,"discussionTemplate":null,"richText":"\u003carticle class=\"markdown-body entry-content container-lg\" itemprop=\"text\"\u003e\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow I Dragged Phantom Tide Out of an OOM Kill Loop\u003c/h1\u003e\u003ca id=\"user-content-how-i-dragged-phantom-tide-out-of-an-oom-kill-loop\" class=\"anchor\" aria-label=\"Permalink: How I Dragged Phantom Tide Out of an OOM Kill Loop\" href=\"#how-i-dragged-phantom-tide-out-of-an-oom-kill-loop\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThere is a certain kind of technical failure that tells you more about a platform than any polished launch thread ever will.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNot when the UI looks clean. Not when the homepage loads fast. Not when the architecture diagram is neatly colour-coded and everyone is pretending the system is more deliberate than it really is. The real test is what happens when production turns hostile, memory gets tight, startup paths collide, caches swell, schedulers stampede, and the kernel starts killing your process like it has lost patience with your design.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat was the point where Phantom Tide became real.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eFrom the outside, the symptom was boring: 502s across the site. From the inside, it was a systems failure spread across FastAPI, uvicorn, Redis, ClickHouse, APScheduler, Docker memory limits, and a startup sequence that had quietly become a deterministic self-attack.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWhat follows is the technical story of how I fixed it. But it is also the real story of what Phantom Tide actually is.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBecause the uncomfortable truth is this: I did not build a map.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe map is just the visible surface. The real work was underneath it: source ingestion, event normalisation, scheduler behaviour, memory control, failure isolation, storage trade-offs, and the ugly operational logic required to make hostile upstream data behave like one coherent system. If you only look at Phantom Tide as a frontend product, you miss the engineering entirely. The map is just the proof that the machinery resolves into something legible. The machinery is the point.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eI thought I was fixing a crash loop. I was really exposing the architecture.\u003c/h2\u003e\u003ca id=\"user-content-i-thought-i-was-fixing-a-crash-loop-i-was-really-exposing-the-architecture\" class=\"anchor\" aria-label=\"Permalink: I thought I was fixing a crash loop. I was really exposing the architecture.\" href=\"#i-thought-i-was-fixing-a-crash-loop-i-was-really-exposing-the-architecture\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAt first glance this looked like an ordinary outage. Phantom Tide started returning 502 Bad Gateway responses and the application container was restart-looping. That is the sort of thing a lot of people write off as “server trouble” until they burn hours chasing reverse proxies and restart counters.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eI ignored the polite lies and went straight to the kernel.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003ccode\u003edmesg\u003c/code\u003e made it clear almost immediately that this was not a routing issue or a generic application fault. The Python process was being killed by the OOM killer. The key signal was \u003ccode\u003eanon-rss\u003c/code\u003e, not total VM size, not Docker restart metadata, not container health status. Actual resident anonymous memory was blowing through the limit and the kernel was enforcing it.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat matters because once the kernel is killing your process, the question stops being “why is the site down” and becomes “which part of my system thinks it deserves this much memory, and why did I let it.”\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is the difference between debugging the symptom and debugging the system.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe first problem: I was treating a stateful process like a stateless web app\u003c/h2\u003e\u003ca id=\"user-content-the-first-problem-i-was-treating-a-stateful-process-like-a-stateless-web-app\" class=\"anchor\" aria-label=\"Permalink: The first problem: I was treating a stateful process like a stateless web app\" href=\"#the-first-problem-i-was-treating-a-stateful-process-like-a-stateless-web-app\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe initial deployment was running uvicorn with four workers.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is normal if your application is basically stateless request handling and you want cheap concurrency. It is a bad idea when your process contains mutable in-memory state, a scheduler, startup hydration logic, live buffers, and background collectors that assume singleton behaviour.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIn that setup, four workers do not mean one application serving four times more efficiently. It means four copies of the same process state, four schedulers, four startup passes, four independent memory footprints, and four ways to multiply every mistake hidden in your boot logic.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is not resilience. That is replication of instability.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eSo the first emergency change was brutally simple:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"uvicorn phantom_app.main:app --host 0.0.0.0 --port 8000 --workers 1\"\u003e\u003cpre\u003euvicorn phantom_app.main:app --host 0.0.0.0 --port 8000 --workers 1\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat did not solve the root cause. What it did was stop the platform from amplifying it. Sometimes the first real fix is not elegant. It is just removing the part that is making everything else harder to see.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis was also the first reminder of what Phantom Tide actually proves. Not that I can wire together a few services and make a dashboard. That I understand when architectural defaults stop being defaults and start becoming liabilities.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe second problem: 28 MB in Redis became roughly 2 GB in Python\u003c/h2\u003e\u003ca id=\"user-content-the-second-problem-28-mb-in-redis-became-roughly-2-gb-in-python\" class=\"anchor\" aria-label=\"Permalink: The second problem: 28 MB in Redis became roughly 2 GB in Python\" href=\"#the-second-problem-28-mb-in-redis-became-roughly-2-gb-in-python\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eOnce the worker count dropped, the next obvious suspect was Redis.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe live event set contained around 20,000 records. The backing JSON footprint was roughly 28 MB. That sounds harmless if you think in terms of storage size. It is not harmless if you understand what Python does to structured data once you fully materialise it into live objects.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eOn startup, the app was hydrating the event store by deserialising every record from the Redis sorted set and constructing dataclass instances for each one. Each \u003ccode\u003eEventRecord\u003c/code\u003e carried around 22 fields plus nested structures such as attributes, geometry, and raw source data. That may look compact in JSON. It is not compact in memory.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eJSON is just bytes. Python objects are a managed heap full of per-object overhead, instance dictionaries, nested dictionaries, strings, references, allocator fragmentation, and garbage collector metadata. So a few dozen megabytes of stored data turned into a giant in-process object graph.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe result was approximately a 70x inflation from Redis JSON size to live heap use.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis is exactly the sort of thing that never shows up in superficial product writeups. You can talk all day about “real-time intelligence fusion,” but eventually the truth comes down to ugly details like whether your startup path is quietly reconstructing two gigabytes of Python objects because you were lazy about hydration strategy.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat was the next fix.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eFirst I reduced retention. Then I enforced the limit at hydration time. Then I stopped eagerly materialising the entire backlog as one big in-memory list.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe retention and load limits became explicit:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"MAX_REDIS_EVENTS = 8_000\nredis_store.load_recent_events(hours=24, limit=8_000)\nch_store.load_recent_events(hours=24, limit=8_000)\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c1\"\u003eMAX_REDIS_EVENTS\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e8_000\u003c/span\u003e\n\u003cspan class=\"pl-s1\"\u003eredis_store\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eload_recent_events\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ehours\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e24\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003elimit\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e8_000\u003c/span\u003e)\n\u003cspan class=\"pl-s1\"\u003ech_store\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eload_recent_events\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ehours\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e24\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003elimit\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e8_000\u003c/span\u003e)\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eLive Redis was trimmed immediately so the next restart would not drag the old problem straight back in:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"docker exec phantom_redis redis-cli ZREMRANGEBYRANK phantom:events 0 -8001\"\u003e\u003cpre\u003edocker \u003cspan class=\"pl-c1\"\u003eexec\u003c/span\u003e phantom_redis redis-cli ZREMRANGEBYRANK phantom:events 0 -8001\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThen I changed the hydration path so it streamed records instead of building a giant list up front:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"def hydrate_from_redis_iter(redis_conn):\n for _, raw in redis_conn.zscan_iter('phantom:events'):\n yield EventRecord(**json.loads(raw))\n\ndef insert_streamed(client, it, chunk_size=1000):\n buffer = []\n for rec in it:\n buffer.append(rec.to_tuple())\n if len(buffer) \u0026gt;= chunk_size:\n client.insert('events', buffer)\n buffer.clear()\n if buffer:\n client.insert('events', buffer)\"\u003e\u003cpre\u003e\u003cspan class=\"pl-k\"\u003edef\u003c/span\u003e \u003cspan class=\"pl-en\"\u003ehydrate_from_redis_iter\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eredis_conn\u003c/span\u003e):\n \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003e_\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003eraw\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003ein\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003eredis_conn\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003ezscan_iter\u003c/span\u003e(\u003cspan class=\"pl-s\"\u003e'phantom:events'\u003c/span\u003e):\n \u003cspan class=\"pl-k\"\u003eyield\u003c/span\u003e \u003cspan class=\"pl-en\"\u003eEventRecord\u003c/span\u003e(\u003cspan class=\"pl-c1\"\u003e**\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ejson\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eloads\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eraw\u003c/span\u003e))\n\n\u003cspan class=\"pl-k\"\u003edef\u003c/span\u003e \u003cspan class=\"pl-en\"\u003einsert_streamed\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eclient\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003eit\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003echunk_size\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e1000\u003c/span\u003e):\n \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e []\n \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003erec\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003ein\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003eit\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eappend\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003erec\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eto_tuple\u003c/span\u003e())\n \u003cspan class=\"pl-k\"\u003eif\u003c/span\u003e \u003cspan class=\"pl-en\"\u003elen\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e) \u003cspan class=\"pl-c1\"\u003e\u0026gt;=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003echunk_size\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003eclient\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003einsert\u003c/span\u003e(\u003cspan class=\"pl-s\"\u003e'events'\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e)\n \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eclear\u003c/span\u003e()\n \u003cspan class=\"pl-k\"\u003eif\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003eclient\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003einsert\u003c/span\u003e(\u003cspan class=\"pl-s\"\u003e'events'\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ebuffer\u003c/span\u003e)\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eI also reduced per-object overhead on the hot path with slots-based dataclasses:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"@dataclass(slots=True)\nclass EventRecord:\n ...\"\u003e\u003cpre\u003e\u003cspan class=\"pl-en\"\u003e@\u003cspan class=\"pl-en\"\u003edataclass\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eslots\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003eTrue\u003c/span\u003e)\u003c/span\u003e\n\u003cspan class=\"pl-k\"\u003eclass\u003c/span\u003e \u003cspan class=\"pl-v\"\u003eEventRecord\u003c/span\u003e:\n ...\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat did not magically make Python cheap. It did make the startup path stop behaving like a memory ambush. Staging memray runs showed a material drop in peak usage on the hydration path after streaming and \u003ccode\u003eslots=True\u003c/code\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAgain, this is the part most people bury because it is not glamorous. I think it is the most convincing part of the whole project. The engineering value in Phantom Tide is not that it displays events on a map. It is that it takes badly behaved, heterogeneous upstream data and forces it through a system that can be reasoned about, profiled, and corrected under production pressure.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe third problem: the scheduler was launching a coordinated attack on startup\u003c/h2\u003e\u003ca id=\"user-content-the-third-problem-the-scheduler-was-launching-a-coordinated-attack-on-startup\" class=\"anchor\" aria-label=\"Permalink: The third problem: the scheduler was launching a coordinated attack on startup\" href=\"#the-third-problem-the-scheduler-was-launching-a-coordinated-attack-on-startup\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAfter reducing worker multiplication and fixing the worst of the hydration path, the platform still died.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe important clue was timing. The app would boot, sit at a few hundred megabytes, look stable, and then die exactly thirty seconds later.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNot roughly. Exactly.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat told me this was not a vague leak or gradual accumulation problem. It was a deterministic startup event.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe culprit was APScheduler.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eTo ensure the platform had fresh data as soon as it started, the scheduler registered one-shot startup jobs for the slower collectors before their normal intervals began. That part was fine. The bad part was that all of them were assigned the same \u003ccode\u003enext_run_time\u003c/code\u003e.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"startup_run_time = datetime.now() + timedelta(seconds=30)\n\nfor key, fn, _interval, job_id, job_name, startup_run in slow_jobs:\n collector = popped.get(key)\n if collector is not None and startup_run:\n _scheduler.add_job(\n fn,\n args=[collector],\n id=f\u0026quot;collectors_{job_id}_startup\u0026quot;,\n name=f\u0026quot;Initial {job_name} run\u0026quot;,\n next_run_time=startup_run_time,\n )\"\u003e\u003cpre\u003e\u003cspan class=\"pl-s1\"\u003estartup_run_time\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003edatetime\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003enow\u003c/span\u003e() \u003cspan class=\"pl-c1\"\u003e+\u003c/span\u003e \u003cspan class=\"pl-en\"\u003etimedelta\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eseconds\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e30\u003c/span\u003e)\n\n\u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ekey\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003efn\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003e_interval\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ejob_id\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ejob_name\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003estartup_run\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003ein\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003eslow_jobs\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003epopped\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eget\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ekey\u003c/span\u003e)\n \u003cspan class=\"pl-k\"\u003eif\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e\u003cspan class=\"pl-c1\"\u003eis\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003enot\u003c/span\u003e\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003eNone\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003eand\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003estartup_run\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003e_scheduler\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eadd_job\u003c/span\u003e(\n \u003cspan class=\"pl-s1\"\u003efn\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003eargs\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e[\u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e],\n \u003cspan class=\"pl-s1\"\u003eid\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s\"\u003ef\"collectors_\u003cspan class=\"pl-s1\"\u003e\u003cspan class=\"pl-kos\"\u003e{\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ejob_id\u003c/span\u003e\u003cspan class=\"pl-kos\"\u003e}\u003c/span\u003e\u003c/span\u003e_startup\"\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003ename\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s\"\u003ef\"Initial \u003cspan class=\"pl-s1\"\u003e\u003cspan class=\"pl-kos\"\u003e{\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ejob_name\u003c/span\u003e\u003cspan class=\"pl-kos\"\u003e}\u003c/span\u003e\u003c/span\u003e run\"\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003enext_run_time\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003estartup_run_time\u003c/span\u003e,\n )\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eSo at T+30 seconds, a whole wall of collectors hit the system simultaneously.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNDBC. MARAD. ICC-CCS. GPS advisory sources. USGS. ECCC. VIIRS. NGA MIS. DART. Multiple network fetches, parsers, object allocations, and store writes all landing at once on a constrained VM.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is not startup. That is a deterministic burst load disguised as startup.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis is where the broader product story matters. Phantom Tide is not just “aggregating feeds.” It is a distributed argument about reality across sources that disagree, drift, stall, overproduce, and break in different ways. The engineering challenge is not collecting the signals. It is making those signals survive contact with a live system. Startup scheduling is part of that. If your collectors all fire as one synchronized burst, you have built a fragility engine, not an intelligence platform.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe fix was to introduce jitter and spread the one-shot jobs across a startup window:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"startup_delay = int(os.getenv(\u0026quot;STARTUP_COLLECT_DELAY_SECONDS\u0026quot;, \u0026quot;30\u0026quot;))\nstartup_window = int(os.getenv(\u0026quot;STARTUP_COLLECT_WINDOW_SECONDS\u0026quot;, \u0026quot;180\u0026quot;))\n\ndef _staggered_time(base_delay: int, window: int) -\u0026gt; datetime:\n offset = random.randint(0, max(0, window))\n return datetime.now() + timedelta(seconds=base_delay + offset)\n\nfor key, fn, _interval, job_id, job_name, startup_run in slow_jobs:\n collector = popped.get(key)\n if collector is not None and startup_run:\n _scheduler.add_job(\n fn,\n args=[collector],\n id=f\u0026quot;collectors_{job_id}_startup\u0026quot;,\n name=f\u0026quot;Initial {job_name} run\u0026quot;,\n next_run_time=_staggered_time(startup_delay, startup_window),\n )\"\u003e\u003cpre\u003e\u003cspan class=\"pl-s1\"\u003estartup_delay\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-en\"\u003eint\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eos\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003egetenv\u003c/span\u003e(\u003cspan class=\"pl-s\"\u003e\"STARTUP_COLLECT_DELAY_SECONDS\"\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\"30\"\u003c/span\u003e))\n\u003cspan class=\"pl-s1\"\u003estartup_window\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-en\"\u003eint\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eos\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003egetenv\u003c/span\u003e(\u003cspan class=\"pl-s\"\u003e\"STARTUP_COLLECT_WINDOW_SECONDS\"\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\"180\"\u003c/span\u003e))\n\n\u003cspan class=\"pl-k\"\u003edef\u003c/span\u003e \u003cspan class=\"pl-en\"\u003e_staggered_time\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ebase_delay\u003c/span\u003e: \u003cspan class=\"pl-smi\"\u003eint\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ewindow\u003c/span\u003e: \u003cspan class=\"pl-smi\"\u003eint\u003c/span\u003e) \u003cspan class=\"pl-c1\"\u003e-\u0026gt;\u003c/span\u003e \u003cspan class=\"pl-smi\"\u003edatetime\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003eoffset\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003erandom\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003erandint\u003c/span\u003e(\u003cspan class=\"pl-c1\"\u003e0\u003c/span\u003e, \u003cspan class=\"pl-en\"\u003emax\u003c/span\u003e(\u003cspan class=\"pl-c1\"\u003e0\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ewindow\u003c/span\u003e))\n \u003cspan class=\"pl-k\"\u003ereturn\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003edatetime\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003enow\u003c/span\u003e() \u003cspan class=\"pl-c1\"\u003e+\u003c/span\u003e \u003cspan class=\"pl-en\"\u003etimedelta\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003eseconds\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ebase_delay\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e+\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003eoffset\u003c/span\u003e)\n\n\u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ekey\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003efn\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003e_interval\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ejob_id\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003ejob_name\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003estartup_run\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003ein\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003eslow_jobs\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003epopped\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eget\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ekey\u003c/span\u003e)\n \u003cspan class=\"pl-k\"\u003eif\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e\u003cspan class=\"pl-c1\"\u003eis\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003enot\u003c/span\u003e\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003eNone\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003eand\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003estartup_run\u003c/span\u003e:\n \u003cspan class=\"pl-s1\"\u003e_scheduler\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003eadd_job\u003c/span\u003e(\n \u003cspan class=\"pl-s1\"\u003efn\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003eargs\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e[\u003cspan class=\"pl-s1\"\u003ecollector\u003c/span\u003e],\n \u003cspan class=\"pl-s1\"\u003eid\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s\"\u003ef\"collectors_\u003cspan class=\"pl-s1\"\u003e\u003cspan class=\"pl-kos\"\u003e{\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ejob_id\u003c/span\u003e\u003cspan class=\"pl-kos\"\u003e}\u003c/span\u003e\u003c/span\u003e_startup\"\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003ename\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-s\"\u003ef\"Initial \u003cspan class=\"pl-s1\"\u003e\u003cspan class=\"pl-kos\"\u003e{\u003c/span\u003e\u003cspan class=\"pl-s1\"\u003ejob_name\u003c/span\u003e\u003cspan class=\"pl-kos\"\u003e}\u003c/span\u003e\u003c/span\u003e run\"\u003c/span\u003e,\n \u003cspan class=\"pl-s1\"\u003enext_run_time\u003c/span\u003e\u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e\u003cspan class=\"pl-en\"\u003e_staggered_time\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003estartup_delay\u003c/span\u003e, \u003cspan class=\"pl-s1\"\u003estartup_window\u003c/span\u003e),\n )\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat turned the startup profile from a cliff into a slope. Instead of thirteen collectors hitting together, they spread out over a few minutes. Memory rose gradually. The platform stayed up.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis was one of the most important fixes because it exposed a deeper truth: most real-world system failures are not a result of one dramatic bug. They come from individually reasonable choices interacting at exactly the wrong point in time.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe fourth problem: the fast loop was still colliding with the slow startup window\u003c/h2\u003e\u003ca id=\"user-content-the-fourth-problem-the-fast-loop-was-still-colliding-with-the-slow-startup-window\" class=\"anchor\" aria-label=\"Permalink: The fourth problem: the fast loop was still colliding with the slow startup window\" href=\"#the-fourth-problem-the-fast-loop-was-still-colliding-with-the-slow-startup-window\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe first scheduler fix removed the mass simultaneous burst. It did not fully remove startup overlap.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe fast-loop collectors still had their own initial run scheduled too early, which meant they could still collide with slow collectors that happened to land near the front of the stagger window.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eSo the startup sequence needed a second pass. The fast loop had to move to the end of the slow-collector window instead of overlapping with it. That changed startup from “partially staggered chaos” into an actual controlled sequence.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe rough shape became:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eapplication boots\u003c/li\u003e\n\u003cli\u003escheduler starts\u003c/li\u003e\n\u003cli\u003eslow collectors begin after the base delay, staggered across the window\u003c/li\u003e\n\u003cli\u003eonly after that window closes does the fast-loop initial run fire\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThat sounds small. It was not small. It removed another artificial pressure point and made startup behave like something designed, not something improvised.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe fifth problem: ClickHouse was configured like it owned the box\u003c/h2\u003e\u003ca id=\"user-content-the-fifth-problem-clickhouse-was-configured-like-it-owned-the-box\" class=\"anchor\" aria-label=\"Permalink: The fifth problem: ClickHouse was configured like it owned the box\" href=\"#the-fifth-problem-clickhouse-was-configured-like-it-owned-the-box\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAt the same time, ClickHouse was competing for memory on the same host.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat mattered because the box was shared. Phantom Tide was not running in isolation. Redis, ClickHouse, the app container, and supporting services were all drawing from the same physical machine. ClickHouse 26.2 also has opinions about startup allocations and where certain memory-related settings actually belong.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis turned into a small piece of config archaeology. Some settings were overly aggressive for the environment. Some belonged in different config scopes. Some combinations created ugly startup behaviour. One XML mistake introduced exactly the sort of pointless delay you deserve when editing config under stress.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe path out was iterative and boring in the right way:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ereduce cache sizes\u003c/li\u003e\n\u003cli\u003evalidate XML\u003c/li\u003e\n\u003cli\u003estrip back custom config\u003c/li\u003e\n\u003cli\u003ereintroduce only what is proven safe\u003c/li\u003e\n\u003cli\u003eobserve startup under cgroup limits\u003c/li\u003e\n\u003cli\u003estop treating ClickHouse like the host exists purely for it\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThat mattered because the platform’s strongest engineering story is not “I know how to install ClickHouse.” It is “I know how to operate a mixed system where each component lies differently under stress, and I know how to back the whole thing down to something stable without guessing.”\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIf you want to show engineering skill, this is where it lives. Not in saying you used ClickHouse. In showing you understand when ClickHouse becomes the wrong neighbour on a tight box unless you discipline it.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe sixth problem: huge flushes were creating secondary memory spikes in ClickHouse\u003c/h2\u003e\u003ca id=\"user-content-the-sixth-problem-huge-flushes-were-creating-secondary-memory-spikes-in-clickhouse\" class=\"anchor\" aria-label=\"Permalink: The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse\" href=\"#the-sixth-problem-huge-flushes-were-creating-secondary-memory-spikes-in-clickhouse\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe in-memory store had another bad habit. It was flushing pending events to ClickHouse as one large batch.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat meant after a noisy restart or backlog recovery, the first insert could be enormous. ClickHouse would then need to process that insert in memory before writing it out, which is exactly the kind of thing you do not want happening on a machine already recovering from pressure.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBefore, it effectively looked like this:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"ch_store.insert_events(batch)\"\u003e\u003cpre\u003e\u003cspan class=\"pl-s1\"\u003ech_store\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003einsert_events\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ebatch\u003c/span\u003e)\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat is fine until \u003ccode\u003ebatch\u003c/code\u003e is not fine.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eSo I chunked it:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-python notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"_CH_FLUSH_CHUNK = 2_000\n\nfor i in range(0, len(batch), _CH_FLUSH_CHUNK):\n chunk = batch[i:i + _CH_FLUSH_CHUNK]\n ch_store.insert_events(chunk)\"\u003e\u003cpre\u003e\u003cspan class=\"pl-s1\"\u003e_CH_FLUSH_CHUNK\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e2_000\u003c/span\u003e\n\n\u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ei\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003ein\u003c/span\u003e \u003cspan class=\"pl-en\"\u003erange\u003c/span\u003e(\u003cspan class=\"pl-c1\"\u003e0\u003c/span\u003e, \u003cspan class=\"pl-en\"\u003elen\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003ebatch\u003c/span\u003e), \u003cspan class=\"pl-s1\"\u003e_CH_FLUSH_CHUNK\u003c/span\u003e):\n \u003cspan class=\"pl-s1\"\u003echunk\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e=\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003ebatch\u003c/span\u003e[\u003cspan class=\"pl-s1\"\u003ei\u003c/span\u003e:\u003cspan class=\"pl-s1\"\u003ei\u003c/span\u003e \u003cspan class=\"pl-c1\"\u003e+\u003c/span\u003e \u003cspan class=\"pl-s1\"\u003e_CH_FLUSH_CHUNK\u003c/span\u003e]\n \u003cspan class=\"pl-s1\"\u003ech_store\u003c/span\u003e.\u003cspan class=\"pl-c1\"\u003einsert_events\u003c/span\u003e(\u003cspan class=\"pl-s1\"\u003echunk\u003c/span\u003e)\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat bounded per-insert memory pressure and made recovery behaviour much more predictable.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat kind of change matters more than people think. Predictability is one of the few real luxuries you can buy in production. Large single-shot flushes feel efficient until they are the thing that tips a struggling system over the edge.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe seventh problem: zero swap headroom made every spike fatal\u003c/h2\u003e\u003ca id=\"user-content-the-seventh-problem-zero-swap-headroom-made-every-spike-fatal\" class=\"anchor\" aria-label=\"Permalink: The seventh problem: zero swap headroom made every spike fatal\" href=\"#the-seventh-problem-zero-swap-headroom-made-every-spike-fatal\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe container limits were also too sharp.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe app had effectively no swap breathing room:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-yaml notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"mem_limit: 2g\nmemswap_limit: 2g\"\u003e\u003cpre\u003e\u003cspan class=\"pl-ent\"\u003emem_limit\u003c/span\u003e: \u003cspan class=\"pl-c1\"\u003e2g\u003c/span\u003e\n\u003cspan class=\"pl-ent\"\u003ememswap_limit\u003c/span\u003e: \u003cspan class=\"pl-c1\"\u003e2g\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThat means any transient spike above 2 GB becomes an immediate death sentence. For a service with a rough startup path and bursty collector behaviour, that is needlessly unforgiving.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eOnce the real fixes were in place, I widened the operating margin:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-yaml notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"mem_limit: 4g\nmemswap_limit: 5g\"\u003e\u003cpre\u003e\u003cspan class=\"pl-ent\"\u003emem_limit\u003c/span\u003e: \u003cspan class=\"pl-c1\"\u003e4g\u003c/span\u003e\n\u003cspan class=\"pl-ent\"\u003ememswap_limit\u003c/span\u003e: \u003cspan class=\"pl-c1\"\u003e5g\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis did not replace proper engineering. It complemented it. There is a difference between using headroom to hide a bad design and using headroom to stop temporary spikes from killing an otherwise stabilised service. This was the latter.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat changed after the fixes\u003c/h2\u003e\u003ca id=\"user-content-what-changed-after-the-fixes\" class=\"anchor\" aria-label=\"Permalink: What changed after the fixes\" href=\"#what-changed-after-the-fixes\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAfter the worker reduction, Redis cap, streamed hydration, scheduler staggering, fast-loop sequencing, ClickHouse insert chunking, conservative ClickHouse tuning, and memory headroom adjustments, the platform stopped behaving like a brittle prototype and started behaving like an actual system.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe important thing was not just that it stopped crashing. It was how it stopped crashing.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBefore:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003estartup looked normal and then detonated\u003c/li\u003e\n\u003cli\u003eall collectors behaved like they had no awareness of each other\u003c/li\u003e\n\u003cli\u003eRedis hydration inflated into a giant Python heap\u003c/li\u003e\n\u003cli\u003eClickHouse was competing too aggressively for memory\u003c/li\u003e\n\u003cli\u003eone large persistence flush could create another spike\u003c/li\u003e\n\u003cli\u003ethe container limits gave no room for transient pressure\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eAfter:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003estartup memory climbed gradually instead of cliffing\u003c/li\u003e\n\u003cli\u003ecollectors came online in a controlled sequence\u003c/li\u003e\n\u003cli\u003ehydration was bounded and streamed\u003c/li\u003e\n\u003cli\u003eClickHouse stopped acting like it owned the host\u003c/li\u003e\n\u003cli\u003eflush behaviour became predictable\u003c/li\u003e\n\u003cli\u003ethe app settled into a sane steady-state range\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThat is the real engineering story of Phantom Tide. Not that it fuses AIS, OpenSky, NOTAMs, GPS interference, advisories, marine feeds, and other operational sources into one view. The stronger claim is that it does that while surviving contact with production realities that kill weaker systems.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat this incident proved about the product\u003c/h2\u003e\u003ca id=\"user-content-what-this-incident-proved-about-the-product\" class=\"anchor\" aria-label=\"Permalink: What this incident proved about the product\" href=\"#what-this-incident-proved-about-the-product\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis outage ended up clarifying the product better than any launch copy could have.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eI had been tempted, like everyone is, to talk about Phantom Tide as a live map or an intelligence interface. That is the easy description. It is also the shallow one.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe deeper and more defensible framing is this:\u003c/p\u003e\n\u003cp dir=\"auto\"\u003ePhantom Tide is not a dashboard. It is a systems engineering case study disguised as an intelligence platform.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe difficult part is not rendering events. The difficult part is deciding what deserves to become an event in the first place, how long it should live, how it should be structured, how conflicting sources should coexist, how startup should behave, how persistence should fail safely, and how the platform should degrade under pressure without becoming fiction.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is what the OOM incident surfaced. The value is not the glossy layer. It is the logic and discipline beneath it.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eA year from now, the thing I would regret most is underselling that by talking too much about the output and not enough about the machinery. The map is useful because the platform behind it earned the right to produce it. That is the part worth showing.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe uncomfortable truth\u003c/h2\u003e\u003ca id=\"user-content-the-uncomfortable-truth\" class=\"anchor\" aria-label=\"Permalink: The uncomfortable truth\" href=\"#the-uncomfortable-truth\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIf I marketed Phantom Tide mainly as “a live map that merges these sources,” a lot of technically serious people would underestimate it instantly.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThey would assume the hard part was UI composition and data plumbing. They would file it mentally under dashboard-builder. They would miss the actual signal: scheduler design, source reconciliation, object inflation, memory profiling, batch persistence strategy, cache-vs-archive reasoning, and live operational debugging under constrained infrastructure.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is why I think the mess is the moat.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNot the mess in the sense of sloppy engineering. The mess in the sense of the real problems you only meet when a system is no longer theoretical. Bad upstreams. Structural drift. startup stampedes. false health. silent data loss. stateful assumptions hiding inside supposedly simple services. Those are not embarrassing side notes. They are the proof of contact with reality.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAnyone can make a pretty interface over commodity data. Very few people can explain why 28 MB in Redis became 2 GB in Python, why four workers made the system worse, why startup jitter mattered more than another CPU, why ClickHouse needed discipline, and why archival persistence was still the next priority even after the platform looked stable again.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is the engineering skill. That is what Phantom Tide really demonstrates.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat still remains\u003c/h2\u003e\u003ca id=\"user-content-what-still-remains\" class=\"anchor\" aria-label=\"Permalink: What still remains\" href=\"#what-still-remains\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eFixing the OOM loop did not mean the platform was finished.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe highest-priority remaining issue is ClickHouse insert failure caused by malformed field types. At least one collector is emitting a \u003ccode\u003edict\u003c/code\u003e where ClickHouse expects a string or numeric field, which poisons entire insert batches. The in-memory store and Redis make the platform look healthy in the short term, but the archive path is not trustworthy until there is schema validation at the boundary.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is the next real job:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003evalidate per-event payloads before persistence\u003c/li\u003e\n\u003cli\u003elog the source and field causing schema failures\u003c/li\u003e\n\u003cli\u003ecoerce where appropriate\u003c/li\u003e\n\u003cli\u003ereject where necessary\u003c/li\u003e\n\u003cli\u003estop silent data loss from hiding behind a healthy-looking UI\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThere are other source-specific issues as well, but those are normal platform work. The insert validation gap is the big one because it attacks trust in the archive.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe real lesson\u003c/h2\u003e\u003ca id=\"user-content-the-real-lesson\" class=\"anchor\" aria-label=\"Permalink: The real lesson\" href=\"#the-real-lesson\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe kernel told the truth first.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eEverything else was just me catching up to it.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis was not one bug. It was a layered systems failure where worker topology, object modelling, startup choreography, batch behaviour, database configuration, and memory limits all interacted in exactly the wrong way. The platform was not failing because of a single bad line of code. It was failing because multiple “reasonable” decisions had combined into a design that only looked solid until production put pressure on it.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is why I think this incident matters.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNot because the site went down. Systems go down.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIt matters because it exposed what Phantom Tide actually is: not a map, not a feed aggregator, not a dressed-up frontend project, but a live operational system built on top of messy, high-friction data, where the hardest work is imposing structure, discipline, and survivability on things that do not want to behave.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThat is the engineering story.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAnd that is the part worth selling.\u003c/p\u003e\n\u003c/article\u003e","richTextTruncated":false,"renderedFileInfo":null,"symbols":{"timed_out":false,"not_analyzed":false,"symbols":[{"name":"How I Dragged Phantom Tide Out of an OOM Kill Loop","fully_qualified_name":"How I Dragged Phantom Tide Out of an OOM Kill Loop","kind":"section_1","ident_start":2,"ident_end":52,"extent_start":0,"extent_end":20943,"ident_utf16":{"start":{"line_number":0,"utf16_col":2},"end":{"line_number":0,"utf16_col":52}},"extent_utf16":{"start":{"line_number":0,"utf16_col":0},"end":{"line_number":350,"utf16_col":0}}},{"name":"I thought I was fixing a crash loop. I was really exposing the architecture.","fully_qualified_name":"I thought I was fixing a crash loop. I was really exposing the architecture.","kind":"section_2","ident_start":1597,"ident_end":1673,"extent_start":1594,"extent_end":2704,"ident_utf16":{"start":{"line_number":16,"utf16_col":3},"end":{"line_number":16,"utf16_col":79}},"extent_utf16":{"start":{"line_number":16,"utf16_col":0},"end":{"line_number":28,"utf16_col":0}}},{"name":"The first problem: I was treating a stateful process like a stateless web app","fully_qualified_name":"The first problem: I was treating a stateful process like a stateless web app","kind":"section_2","ident_start":2707,"ident_end":2784,"extent_start":2704,"extent_end":4058,"ident_utf16":{"start":{"line_number":28,"utf16_col":3},"end":{"line_number":28,"utf16_col":80}},"extent_utf16":{"start":{"line_number":28,"utf16_col":0},"end":{"line_number":48,"utf16_col":0}}},{"name":"The second problem: 28 MB in Redis became roughly 2 GB in Python","fully_qualified_name":"The second problem: 28 MB in Redis became roughly 2 GB in Python","kind":"section_2","ident_start":4061,"ident_end":4125,"extent_start":4058,"extent_end":7394,"ident_utf16":{"start":{"line_number":48,"utf16_col":3},"end":{"line_number":48,"utf16_col":67}},"extent_utf16":{"start":{"line_number":48,"utf16_col":0},"end":{"line_number":110,"utf16_col":0}}},{"name":"The third problem: the scheduler was launching a coordinated attack on startup","fully_qualified_name":"The third problem: the scheduler was launching a coordinated attack on startup","kind":"section_2","ident_start":7397,"ident_end":7475,"extent_start":7394,"extent_end":10701,"ident_utf16":{"start":{"line_number":110,"utf16_col":3},"end":{"line_number":110,"utf16_col":81}},"extent_utf16":{"start":{"line_number":110,"utf16_col":0},"end":{"line_number":173,"utf16_col":0}}},{"name":"The fourth problem: the fast loop was still colliding with the slow startup window","fully_qualified_name":"The fourth problem: the fast loop was still colliding with the slow startup window","kind":"section_2","ident_start":10704,"ident_end":10786,"extent_start":10701,"extent_end":11692,"ident_utf16":{"start":{"line_number":173,"utf16_col":3},"end":{"line_number":173,"utf16_col":85}},"extent_utf16":{"start":{"line_number":173,"utf16_col":0},"end":{"line_number":190,"utf16_col":0}}},{"name":"The fifth problem: ClickHouse was configured like it owned the box","fully_qualified_name":"The fifth problem: ClickHouse was configured like it owned the box","kind":"section_2","ident_start":11695,"ident_end":11761,"extent_start":11692,"extent_end":13234,"ident_utf16":{"start":{"line_number":190,"utf16_col":3},"end":{"line_number":190,"utf16_col":69}},"extent_utf16":{"start":{"line_number":190,"utf16_col":0},"end":{"line_number":211,"utf16_col":0}}},{"name":"The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse","fully_qualified_name":"The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse","kind":"section_2","ident_start":13237,"ident_end":13320,"extent_start":13234,"extent_end":14352,"ident_utf16":{"start":{"line_number":211,"utf16_col":3},"end":{"line_number":211,"utf16_col":86}},"extent_utf16":{"start":{"line_number":211,"utf16_col":0},"end":{"line_number":239,"utf16_col":0}}},{"name":"The seventh problem: zero swap headroom made every spike fatal","fully_qualified_name":"The seventh problem: zero swap headroom made every spike fatal","kind":"section_2","ident_start":14355,"ident_end":14417,"extent_start":14352,"extent_end":15092,"ident_utf16":{"start":{"line_number":239,"utf16_col":3},"end":{"line_number":239,"utf16_col":65}},"extent_utf16":{"start":{"line_number":239,"utf16_col":0},"end":{"line_number":261,"utf16_col":0}}},{"name":"What changed after the fixes","fully_qualified_name":"What changed after the fixes","kind":"section_2","ident_start":15095,"ident_end":15123,"extent_start":15092,"extent_end":16441,"ident_utf16":{"start":{"line_number":261,"utf16_col":3},"end":{"line_number":261,"utf16_col":31}},"extent_utf16":{"start":{"line_number":261,"utf16_col":0},"end":{"line_number":287,"utf16_col":0}}},{"name":"What this incident proved about the product","fully_qualified_name":"What this incident proved about the product","kind":"section_2","ident_start":16444,"ident_end":16487,"extent_start":16441,"extent_end":17640,"ident_utf16":{"start":{"line_number":287,"utf16_col":3},"end":{"line_number":287,"utf16_col":46}},"extent_utf16":{"start":{"line_number":287,"utf16_col":0},"end":{"line_number":303,"utf16_col":0}}},{"name":"The uncomfortable truth","fully_qualified_name":"The uncomfortable truth","kind":"section_2","ident_start":17643,"ident_end":17666,"extent_start":17640,"extent_end":19030,"ident_utf16":{"start":{"line_number":303,"utf16_col":3},"end":{"line_number":303,"utf16_col":26}},"extent_utf16":{"start":{"line_number":303,"utf16_col":0},"end":{"line_number":317,"utf16_col":0}}},{"name":"What still remains","fully_qualified_name":"What still remains","kind":"section_2","ident_start":19033,"ident_end":19051,"extent_start":19030,"extent_end":19929,"ident_utf16":{"start":{"line_number":317,"utf16_col":3},"end":{"line_number":317,"utf16_col":21}},"extent_utf16":{"start":{"line_number":317,"utf16_col":0},"end":{"line_number":333,"utf16_col":0}}},{"name":"The real lesson","fully_qualified_name":"The real lesson","kind":"section_2","ident_start":19932,"ident_end":19947,"extent_start":19929,"extent_end":20943,"ident_utf16":{"start":{"line_number":333,"utf16_col":3},"end":{"line_number":333,"utf16_col":18}},"extent_utf16":{"start":{"line_number":333,"utf16_col":0},"end":{"line_number":350,"utf16_col":0}}}]}},"codeViewLayoutRoute":{"repo":{"id":1188278606,"defaultBranch":"main","name":"phantomtide","ownerLogin":"tg12","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2026-03-21T21:24:39.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/12201893?v=4","public":true,"private":false,"isOrgOwned":false},"currentUser":null,"uploadToken":"xyDRXPNKIPFgjfl_KoAiii-GOjzp2GF-EFzj_Lh_NRKos-9e6fefuhWj-L-7XUQkui7W4NM0vN_lx-iaPG6dhQ","allShortcutsEnabled":false,"treeExpanded":true,"path":"docs/oom-postmortem.md","symbolsExpanded":false,"refInfo":{"name":"main","listCacheKey":"v0:1775764413.0","canEdit":false,"currentOid":"6629adcc6b1bb49d67dfc520997707daf595cf74"},"helpUrl":"https://docs.github.com","githubDevUrl":null},"codeViewFileTreeLayoutRoute":{"fileTree":{"docs":{"items":[{"name":"screenshots","path":"docs/screenshots","contentType":"directory"},{"name":"geojson-cpu-outage.md","path":"docs/geojson-cpu-outage.md","contentType":"file"},{"name":"geojson-cpu-triage.md","path":"docs/geojson-cpu-triage.md","contentType":"file"},{"name":"oom-postmortem.md","path":"docs/oom-postmortem.md","contentType":"file"},{"name":"operator-guide.md","path":"docs/operator-guide.md","contentType":"file"}],"totalCount":5},"":{"items":[{"name":".github","path":".github","contentType":"directory"},{"name":"airspace_crossing_viz","path":"airspace_crossing_viz","contentType":"directory"},{"name":"docs","path":"docs","contentType":"directory"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":"CHANGELOG.md","path":"CHANGELOG.md","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"}],"totalCount":6}},"fileTreeProcessingTime":36.394706,"foldersToFetch":[]},"codeViewBlobLayoutRoute":{"codeLineWrapEnabled":false,"refInfo":{"name":"main","listCacheKey":"v0:1775764413.0","canEdit":false,"refType":"branch","currentOid":"6629adcc6b1bb49d67dfc520997707daf595cf74","canEditOnDefaultBranch":false,"fileExistsOnDefault":true},"path":"docs/oom-postmortem.md","blob":{"copilotSWEAgentEnabled":false,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/tg12/phantomtide/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"oom-postmortem.md","displayUrl":"https://github.com/tg12/phantomtide/blob/main/docs/oom-postmortem.md?raw=true","headerInfo":{"blobSize":"20.5 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"2c4825f","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Ftg12%2Fphantomtide%2Fblob%2Fmain%2Fdocs%252Foom-postmortem.md","isCSV":false,"isRichtext":true,"lineInfo":{"truncatedLoc":"350","truncatedSloc":"217"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"isIssueTemplate":false,"isDiscussionTemplate":false,"language":"Markdown","languageID":222,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/tg12/phantomtide/blob/main/docs%2Foom-postmortem.md","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/tg12/phantomtide/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/tg12/phantomtide/raw/refs/heads/main/docs/oom-postmortem.md","renderImageOrRaw":false,"shortPath":null,"symbolsEnabled":true,"tabSize":4,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null},"copilotInfo":null,"copilotAccessAllowed":false,"copilotSpacesEnabled":false,"modelsAccessAllowed":false,"modelsRepoIntegrationEnabled":false,"isMarketplaceEnabled":true},"codeViewBlobLayoutRoute.StyledBlob":{"rawLines":["# How I Dragged Phantom Tide Out of an OOM Kill Loop","","There is a certain kind of technical failure that tells you more about a platform than any polished launch thread ever will.","","Not when the UI looks clean. Not when the homepage loads fast. Not when the architecture diagram is neatly colour-coded and everyone is pretending the system is more deliberate than it really is. The real test is what happens when production turns hostile, memory gets tight, startup paths collide, caches swell, schedulers stampede, and the kernel starts killing your process like it has lost patience with your design.","","That was the point where Phantom Tide became real.","","From the outside, the symptom was boring: 502s across the site. From the inside, it was a systems failure spread across FastAPI, uvicorn, Redis, ClickHouse, APScheduler, Docker memory limits, and a startup sequence that had quietly become a deterministic self-attack.","","What follows is the technical story of how I fixed it. But it is also the real story of what Phantom Tide actually is.","","Because the uncomfortable truth is this: I did not build a map.","","The map is just the visible surface. The real work was underneath it: source ingestion, event normalisation, scheduler behaviour, memory control, failure isolation, storage trade-offs, and the ugly operational logic required to make hostile upstream data behave like one coherent system. If you only look at Phantom Tide as a frontend product, you miss the engineering entirely. The map is just the proof that the machinery resolves into something legible. The machinery is the point.","","## I thought I was fixing a crash loop. I was really exposing the architecture.","","At first glance this looked like an ordinary outage. Phantom Tide started returning 502 Bad Gateway responses and the application container was restart-looping. That is the sort of thing a lot of people write off as “server trouble” until they burn hours chasing reverse proxies and restart counters.","","I ignored the polite lies and went straight to the kernel.","","`dmesg` made it clear almost immediately that this was not a routing issue or a generic application fault. The Python process was being killed by the OOM killer. The key signal was `anon-rss`, not total VM size, not Docker restart metadata, not container health status. Actual resident anonymous memory was blowing through the limit and the kernel was enforcing it.","","That matters because once the kernel is killing your process, the question stops being “why is the site down” and becomes “which part of my system thinks it deserves this much memory, and why did I let it.”","","That is the difference between debugging the symptom and debugging the system.","","## The first problem: I was treating a stateful process like a stateless web app","","The initial deployment was running uvicorn with four workers.","","That is normal if your application is basically stateless request handling and you want cheap concurrency. It is a bad idea when your process contains mutable in-memory state, a scheduler, startup hydration logic, live buffers, and background collectors that assume singleton behaviour.","","In that setup, four workers do not mean one application serving four times more efficiently. It means four copies of the same process state, four schedulers, four startup passes, four independent memory footprints, and four ways to multiply every mistake hidden in your boot logic.","","That is not resilience. That is replication of instability.","","So the first emergency change was brutally simple:","","```sh","uvicorn phantom_app.main:app --host 0.0.0.0 --port 8000 --workers 1","```","","That did not solve the root cause. What it did was stop the platform from amplifying it. Sometimes the first real fix is not elegant. It is just removing the part that is making everything else harder to see.","","This was also the first reminder of what Phantom Tide actually proves. Not that I can wire together a few services and make a dashboard. That I understand when architectural defaults stop being defaults and start becoming liabilities.","","## The second problem: 28 MB in Redis became roughly 2 GB in Python","","Once the worker count dropped, the next obvious suspect was Redis.","","The live event set contained around 20,000 records. The backing JSON footprint was roughly 28 MB. That sounds harmless if you think in terms of storage size. It is not harmless if you understand what Python does to structured data once you fully materialise it into live objects.","","On startup, the app was hydrating the event store by deserialising every record from the Redis sorted set and constructing dataclass instances for each one. Each `EventRecord` carried around 22 fields plus nested structures such as attributes, geometry, and raw source data. That may look compact in JSON. It is not compact in memory.","","JSON is just bytes. Python objects are a managed heap full of per-object overhead, instance dictionaries, nested dictionaries, strings, references, allocator fragmentation, and garbage collector metadata. So a few dozen megabytes of stored data turned into a giant in-process object graph.","","The result was approximately a 70x inflation from Redis JSON size to live heap use.","","This is exactly the sort of thing that never shows up in superficial product writeups. You can talk all day about “real-time intelligence fusion,” but eventually the truth comes down to ugly details like whether your startup path is quietly reconstructing two gigabytes of Python objects because you were lazy about hydration strategy.","","That was the next fix.","","First I reduced retention. Then I enforced the limit at hydration time. Then I stopped eagerly materialising the entire backlog as one big in-memory list.","","The retention and load limits became explicit:","","```python","MAX_REDIS_EVENTS = 8_000","redis_store.load_recent_events(hours=24, limit=8_000)","ch_store.load_recent_events(hours=24, limit=8_000)","```","","Live Redis was trimmed immediately so the next restart would not drag the old problem straight back in:","","```sh","docker exec phantom_redis redis-cli ZREMRANGEBYRANK phantom:events 0 -8001","```","","Then I changed the hydration path so it streamed records instead of building a giant list up front:","","```python","def hydrate_from_redis_iter(redis_conn):"," for _, raw in redis_conn.zscan_iter('phantom:events'):"," yield EventRecord(**json.loads(raw))","","def insert_streamed(client, it, chunk_size=1000):"," buffer = []"," for rec in it:"," buffer.append(rec.to_tuple())"," if len(buffer) \u003e= chunk_size:"," client.insert('events', buffer)"," buffer.clear()"," if buffer:"," client.insert('events', buffer)","```","","I also reduced per-object overhead on the hot path with slots-based dataclasses:","","```python","@dataclass(slots=True)","class EventRecord:"," ...","```","","That did not magically make Python cheap. It did make the startup path stop behaving like a memory ambush. Staging memray runs showed a material drop in peak usage on the hydration path after streaming and `slots=True`.","","Again, this is the part most people bury because it is not glamorous. I think it is the most convincing part of the whole project. The engineering value in Phantom Tide is not that it displays events on a map. It is that it takes badly behaved, heterogeneous upstream data and forces it through a system that can be reasoned about, profiled, and corrected under production pressure.","","## The third problem: the scheduler was launching a coordinated attack on startup","","After reducing worker multiplication and fixing the worst of the hydration path, the platform still died.","","The important clue was timing. The app would boot, sit at a few hundred megabytes, look stable, and then die exactly thirty seconds later.","","Not roughly. Exactly.","","That told me this was not a vague leak or gradual accumulation problem. It was a deterministic startup event.","","The culprit was APScheduler.","","To ensure the platform had fresh data as soon as it started, the scheduler registered one-shot startup jobs for the slower collectors before their normal intervals began. That part was fine. The bad part was that all of them were assigned the same `next_run_time`.","","```python","startup_run_time = datetime.now() + timedelta(seconds=30)","","for key, fn, _interval, job_id, job_name, startup_run in slow_jobs:"," collector = popped.get(key)"," if collector is not None and startup_run:"," _scheduler.add_job("," fn,"," args=[collector],"," id=f\"collectors_{job_id}_startup\","," name=f\"Initial {job_name} run\","," next_run_time=startup_run_time,"," )","```","","So at T+30 seconds, a whole wall of collectors hit the system simultaneously.","","NDBC. MARAD. ICC-CCS. GPS advisory sources. USGS. ECCC. VIIRS. NGA MIS. DART. Multiple network fetches, parsers, object allocations, and store writes all landing at once on a constrained VM.","","That is not startup. That is a deterministic burst load disguised as startup.","","This is where the broader product story matters. Phantom Tide is not just “aggregating feeds.” It is a distributed argument about reality across sources that disagree, drift, stall, overproduce, and break in different ways. The engineering challenge is not collecting the signals. It is making those signals survive contact with a live system. Startup scheduling is part of that. If your collectors all fire as one synchronized burst, you have built a fragility engine, not an intelligence platform.","","The fix was to introduce jitter and spread the one-shot jobs across a startup window:","","```python","startup_delay = int(os.getenv(\"STARTUP_COLLECT_DELAY_SECONDS\", \"30\"))","startup_window = int(os.getenv(\"STARTUP_COLLECT_WINDOW_SECONDS\", \"180\"))","","def _staggered_time(base_delay: int, window: int) -\u003e datetime:"," offset = random.randint(0, max(0, window))"," return datetime.now() + timedelta(seconds=base_delay + offset)","","for key, fn, _interval, job_id, job_name, startup_run in slow_jobs:"," collector = popped.get(key)"," if collector is not None and startup_run:"," _scheduler.add_job("," fn,"," args=[collector],"," id=f\"collectors_{job_id}_startup\","," name=f\"Initial {job_name} run\","," next_run_time=_staggered_time(startup_delay, startup_window),"," )","```","","That turned the startup profile from a cliff into a slope. Instead of thirteen collectors hitting together, they spread out over a few minutes. Memory rose gradually. The platform stayed up.","","This was one of the most important fixes because it exposed a deeper truth: most real-world system failures are not a result of one dramatic bug. They come from individually reasonable choices interacting at exactly the wrong point in time.","","## The fourth problem: the fast loop was still colliding with the slow startup window","","The first scheduler fix removed the mass simultaneous burst. It did not fully remove startup overlap.","","The fast-loop collectors still had their own initial run scheduled too early, which meant they could still collide with slow collectors that happened to land near the front of the stagger window.","","So the startup sequence needed a second pass. The fast loop had to move to the end of the slow-collector window instead of overlapping with it. That changed startup from “partially staggered chaos” into an actual controlled sequence.","","The rough shape became:","","* application boots","* scheduler starts","* slow collectors begin after the base delay, staggered across the window","* only after that window closes does the fast-loop initial run fire","","That sounds small. It was not small. It removed another artificial pressure point and made startup behave like something designed, not something improvised.","","## The fifth problem: ClickHouse was configured like it owned the box","","At the same time, ClickHouse was competing for memory on the same host.","","That mattered because the box was shared. Phantom Tide was not running in isolation. Redis, ClickHouse, the app container, and supporting services were all drawing from the same physical machine. ClickHouse 26.2 also has opinions about startup allocations and where certain memory-related settings actually belong.","","This turned into a small piece of config archaeology. Some settings were overly aggressive for the environment. Some belonged in different config scopes. Some combinations created ugly startup behaviour. One XML mistake introduced exactly the sort of pointless delay you deserve when editing config under stress.","","The path out was iterative and boring in the right way:","","* reduce cache sizes","* validate XML","* strip back custom config","* reintroduce only what is proven safe","* observe startup under cgroup limits","* stop treating ClickHouse like the host exists purely for it","","That mattered because the platform’s strongest engineering story is not “I know how to install ClickHouse.” It is “I know how to operate a mixed system where each component lies differently under stress, and I know how to back the whole thing down to something stable without guessing.”","","If you want to show engineering skill, this is where it lives. Not in saying you used ClickHouse. In showing you understand when ClickHouse becomes the wrong neighbour on a tight box unless you discipline it.","","## The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse","","The in-memory store had another bad habit. It was flushing pending events to ClickHouse as one large batch.","","That meant after a noisy restart or backlog recovery, the first insert could be enormous. ClickHouse would then need to process that insert in memory before writing it out, which is exactly the kind of thing you do not want happening on a machine already recovering from pressure.","","Before, it effectively looked like this:","","```python","ch_store.insert_events(batch)","```","","That is fine until `batch` is not fine.","","So I chunked it:","","```python","_CH_FLUSH_CHUNK = 2_000","","for i in range(0, len(batch), _CH_FLUSH_CHUNK):"," chunk = batch[i:i + _CH_FLUSH_CHUNK]"," ch_store.insert_events(chunk)","```","","That bounded per-insert memory pressure and made recovery behaviour much more predictable.","","That kind of change matters more than people think. Predictability is one of the few real luxuries you can buy in production. Large single-shot flushes feel efficient until they are the thing that tips a struggling system over the edge.","","## The seventh problem: zero swap headroom made every spike fatal","","The container limits were also too sharp.","","The app had effectively no swap breathing room:","","```yaml","mem_limit: 2g","memswap_limit: 2g","```","","That means any transient spike above 2 GB becomes an immediate death sentence. For a service with a rough startup path and bursty collector behaviour, that is needlessly unforgiving.","","Once the real fixes were in place, I widened the operating margin:","","```yaml","mem_limit: 4g","memswap_limit: 5g","```","","This did not replace proper engineering. It complemented it. There is a difference between using headroom to hide a bad design and using headroom to stop temporary spikes from killing an otherwise stabilised service. This was the latter.","","## What changed after the fixes","","After the worker reduction, Redis cap, streamed hydration, scheduler staggering, fast-loop sequencing, ClickHouse insert chunking, conservative ClickHouse tuning, and memory headroom adjustments, the platform stopped behaving like a brittle prototype and started behaving like an actual system.","","The important thing was not just that it stopped crashing. It was how it stopped crashing.","","Before:","","* startup looked normal and then detonated","* all collectors behaved like they had no awareness of each other","* Redis hydration inflated into a giant Python heap","* ClickHouse was competing too aggressively for memory","* one large persistence flush could create another spike","* the container limits gave no room for transient pressure","","After:","","* startup memory climbed gradually instead of cliffing","* collectors came online in a controlled sequence","* hydration was bounded and streamed","* ClickHouse stopped acting like it owned the host","* flush behaviour became predictable","* the app settled into a sane steady-state range","","That is the real engineering story of Phantom Tide. Not that it fuses AIS, OpenSky, NOTAMs, GPS interference, advisories, marine feeds, and other operational sources into one view. The stronger claim is that it does that while surviving contact with production realities that kill weaker systems.","","## What this incident proved about the product","","This outage ended up clarifying the product better than any launch copy could have.","","I had been tempted, like everyone is, to talk about Phantom Tide as a live map or an intelligence interface. That is the easy description. It is also the shallow one.","","The deeper and more defensible framing is this:","","Phantom Tide is not a dashboard. It is a systems engineering case study disguised as an intelligence platform.","","The difficult part is not rendering events. The difficult part is deciding what deserves to become an event in the first place, how long it should live, how it should be structured, how conflicting sources should coexist, how startup should behave, how persistence should fail safely, and how the platform should degrade under pressure without becoming fiction.","","That is what the OOM incident surfaced. The value is not the glossy layer. It is the logic and discipline beneath it.","","A year from now, the thing I would regret most is underselling that by talking too much about the output and not enough about the machinery. The map is useful because the platform behind it earned the right to produce it. That is the part worth showing.","","## The uncomfortable truth","","If I marketed Phantom Tide mainly as “a live map that merges these sources,” a lot of technically serious people would underestimate it instantly.","","They would assume the hard part was UI composition and data plumbing. They would file it mentally under dashboard-builder. They would miss the actual signal: scheduler design, source reconciliation, object inflation, memory profiling, batch persistence strategy, cache-vs-archive reasoning, and live operational debugging under constrained infrastructure.","","That is why I think the mess is the moat.","","Not the mess in the sense of sloppy engineering. The mess in the sense of the real problems you only meet when a system is no longer theoretical. Bad upstreams. Structural drift. startup stampedes. false health. silent data loss. stateful assumptions hiding inside supposedly simple services. Those are not embarrassing side notes. They are the proof of contact with reality.","","Anyone can make a pretty interface over commodity data. Very few people can explain why 28 MB in Redis became 2 GB in Python, why four workers made the system worse, why startup jitter mattered more than another CPU, why ClickHouse needed discipline, and why archival persistence was still the next priority even after the platform looked stable again.","","That is the engineering skill. That is what Phantom Tide really demonstrates.","","## What still remains","","Fixing the OOM loop did not mean the platform was finished.","","The highest-priority remaining issue is ClickHouse insert failure caused by malformed field types. At least one collector is emitting a `dict` where ClickHouse expects a string or numeric field, which poisons entire insert batches. The in-memory store and Redis make the platform look healthy in the short term, but the archive path is not trustworthy until there is schema validation at the boundary.","","That is the next real job:","","* validate per-event payloads before persistence","* log the source and field causing schema failures","* coerce where appropriate","* reject where necessary","* stop silent data loss from hiding behind a healthy-looking UI","","There are other source-specific issues as well, but those are normal platform work. The insert validation gap is the big one because it attacks trust in the archive.","","## The real lesson","","The kernel told the truth first.","","Everything else was just me catching up to it.","","This was not one bug. It was a layered systems failure where worker topology, object modelling, startup choreography, batch behaviour, database configuration, and memory limits all interacted in exactly the wrong way. The platform was not failing because of a single bad line of code. It was failing because multiple “reasonable” decisions had combined into a design that only looked solid until production put pressure on it.","","That is why I think this incident matters.","","Not because the site went down. Systems go down.","","It matters because it exposed what Phantom Tide actually is: not a map, not a feed aggregator, not a dressed-up frontend project, but a live operational system built on top of messy, high-friction data, where the hardest work is imposing structure, discipline, and survivability on things that do not want to behave.","","That is the engineering story.","","And that is the part worth selling."],"stylingDirectives":[[[0,52,"pl-mh"],[2,52,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[[0,79,"pl-mh"],[3,79,"pl-en"]],[],[],[],[],[],[[0,1,"pl-s"],[1,6,"pl-c1"],[6,7,"pl-s"],[181,182,"pl-s"],[182,190,"pl-c1"],[190,191,"pl-s"]],[],[],[],[],[],[[0,80,"pl-mh"],[3,80,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[],[[0,3,"pl-s"],[3,5,"pl-en"]],[],[[0,3,"pl-s"]],[],[],[],[],[],[[0,67,"pl-mh"],[3,67,"pl-en"]],[],[],[],[],[],[[162,163,"pl-s"],[163,174,"pl-c1"],[174,175,"pl-s"]],[],[],[],[],[],[],[],[],[],[],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[0,16,"pl-c1"],[17,18,"pl-k"],[19,24,"pl-c1"]],[[31,36,"pl-v"],[36,37,"pl-k"],[37,39,"pl-c1"],[41,46,"pl-v"],[46,47,"pl-k"],[47,52,"pl-c1"]],[[28,33,"pl-v"],[33,34,"pl-k"],[34,36,"pl-c1"],[38,43,"pl-v"],[43,44,"pl-k"],[44,49,"pl-c1"]],[[0,3,"pl-s"]],[],[],[],[[0,3,"pl-s"],[3,5,"pl-en"]],[[7,11,"pl-c1"]],[[0,3,"pl-s"]],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[0,3,"pl-k"],[4,27,"pl-en"],[28,38,"pl-smi"]],[[4,7,"pl-k"],[15,17,"pl-k"],[40,56,"pl-s"],[40,41,"pl-pds"],[55,56,"pl-pds"]],[[8,13,"pl-k"],[26,28,"pl-k"]],[],[[0,3,"pl-k"],[4,19,"pl-en"],[20,26,"pl-smi"],[28,30,"pl-smi"],[32,42,"pl-smi"],[42,43,"pl-k"],[43,47,"pl-c1"]],[[11,12,"pl-k"]],[[4,7,"pl-k"],[12,14,"pl-k"]],[],[[8,10,"pl-k"],[11,14,"pl-c1"],[23,25,"pl-k"]],[[26,34,"pl-s"],[26,27,"pl-pds"],[33,34,"pl-pds"]],[],[[4,6,"pl-k"]],[[22,30,"pl-s"],[22,23,"pl-pds"],[29,30,"pl-pds"]],[[0,3,"pl-s"]],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[0,10,"pl-en"],[11,16,"pl-v"],[16,17,"pl-k"],[17,21,"pl-c1"]],[[0,5,"pl-k"],[6,17,"pl-en"]],[[4,7,"pl-c1"]],[[0,3,"pl-s"]],[],[[206,207,"pl-s"],[207,217,"pl-c1"],[217,218,"pl-s"]],[],[],[],[[0,81,"pl-mh"],[3,81,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[],[[248,249,"pl-s"],[249,262,"pl-c1"],[262,263,"pl-s"]],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[17,18,"pl-k"],[34,35,"pl-k"],[46,53,"pl-v"],[53,54,"pl-k"],[54,56,"pl-c1"]],[],[[0,3,"pl-k"],[54,56,"pl-k"]],[[14,15,"pl-k"]],[[4,6,"pl-k"],[17,19,"pl-k"],[20,23,"pl-k"],[24,28,"pl-c1"],[29,32,"pl-k"]],[],[],[[12,16,"pl-v"],[16,17,"pl-k"]],[[12,14,"pl-v"],[14,15,"pl-k"],[15,16,"pl-s"],[16,17,"pl-pds"],[17,28,"pl-s"],[28,29,"pl-c1"],[35,36,"pl-c1"],[36,44,"pl-s"],[44,45,"pl-pds"]],[[12,16,"pl-v"],[16,17,"pl-k"],[17,18,"pl-s"],[18,19,"pl-pds"],[19,27,"pl-s"],[27,28,"pl-c1"],[36,37,"pl-c1"],[37,41,"pl-s"],[41,42,"pl-pds"]],[[12,25,"pl-v"],[25,26,"pl-k"]],[],[[0,3,"pl-s"]],[],[],[],[],[],[],[],[],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[14,15,"pl-k"],[16,19,"pl-c1"],[30,61,"pl-s"],[30,31,"pl-pds"],[60,61,"pl-pds"],[63,67,"pl-s"],[63,64,"pl-pds"],[66,67,"pl-pds"]],[[15,16,"pl-k"],[17,20,"pl-c1"],[31,63,"pl-s"],[31,32,"pl-pds"],[62,63,"pl-pds"],[65,70,"pl-s"],[65,66,"pl-pds"],[69,70,"pl-pds"]],[],[[0,3,"pl-k"],[4,19,"pl-en"],[20,30,"pl-smi"],[32,35,"pl-c1"],[37,43,"pl-smi"],[45,48,"pl-c1"]],[[11,12,"pl-k"],[28,29,"pl-c1"],[31,34,"pl-c1"],[35,36,"pl-c1"]],[[4,10,"pl-k"],[26,27,"pl-k"],[38,45,"pl-v"],[45,46,"pl-k"],[57,58,"pl-k"]],[],[[0,3,"pl-k"],[54,56,"pl-k"]],[[14,15,"pl-k"]],[[4,6,"pl-k"],[17,19,"pl-k"],[20,23,"pl-k"],[24,28,"pl-c1"],[29,32,"pl-k"]],[],[],[[12,16,"pl-v"],[16,17,"pl-k"]],[[12,14,"pl-v"],[14,15,"pl-k"],[15,16,"pl-s"],[16,17,"pl-pds"],[17,28,"pl-s"],[28,29,"pl-c1"],[35,36,"pl-c1"],[36,44,"pl-s"],[44,45,"pl-pds"]],[[12,16,"pl-v"],[16,17,"pl-k"],[17,18,"pl-s"],[18,19,"pl-pds"],[19,27,"pl-s"],[27,28,"pl-c1"],[36,37,"pl-c1"],[37,41,"pl-s"],[41,42,"pl-pds"]],[[12,25,"pl-v"],[25,26,"pl-k"]],[],[[0,3,"pl-s"]],[],[],[],[],[],[[0,85,"pl-mh"],[3,85,"pl-en"]],[],[],[],[],[],[],[],[],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[],[],[[0,69,"pl-mh"],[3,69,"pl-en"]],[],[],[],[],[],[],[],[],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[],[],[],[],[[0,86,"pl-mh"],[3,86,"pl-en"]],[],[],[],[],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[],[[0,3,"pl-s"]],[],[[19,20,"pl-s"],[20,25,"pl-c1"],[25,26,"pl-s"]],[],[],[],[[0,3,"pl-s"],[3,9,"pl-en"]],[[0,15,"pl-c1"],[16,17,"pl-k"],[18,23,"pl-c1"]],[],[[0,3,"pl-k"],[6,8,"pl-k"],[9,14,"pl-c1"],[15,16,"pl-c1"],[18,21,"pl-c1"],[30,45,"pl-c1"]],[[10,11,"pl-k"],[22,23,"pl-k"],[24,39,"pl-c1"]],[],[[0,3,"pl-s"]],[],[],[],[],[],[[0,65,"pl-mh"],[3,65,"pl-en"]],[],[],[],[],[],[[0,3,"pl-s"],[3,7,"pl-en"]],[[0,9,"pl-ent"],[11,13,"pl-c1"]],[[0,13,"pl-ent"],[15,17,"pl-c1"]],[[0,3,"pl-s"],[0,1,"pl-pds"],[1,2,"pl-pds"],[2,3,"pl-pds"]],[[0,0,"pl-s"]],[[0,182,"pl-s"]],[[0,0,"pl-s"]],[[0,66,"pl-s"]],[[0,0,"pl-s"]],[[0,3,"pl-s"],[0,1,"pl-pds"],[1,2,"pl-pds"],[2,3,"pl-pds"],[3,7,"pl-s"]],[[0,9,"pl-ent"],[11,13,"pl-c1"]],[[0,13,"pl-ent"],[15,17,"pl-c1"]],[[0,3,"pl-s"],[0,1,"pl-pds"],[1,2,"pl-pds"],[2,3,"pl-pds"]],[[0,0,"pl-s"]],[[0,237,"pl-s"]],[[0,0,"pl-s"]],[[0,31,"pl-s"]],[[0,0,"pl-s"]],[[0,294,"pl-s"]],[[0,0,"pl-s"]],[[0,90,"pl-s"]],[[0,0,"pl-s"]],[[0,7,"pl-s"]],[[0,0,"pl-s"]],[[0,42,"pl-s"]],[[0,65,"pl-s"]],[[0,51,"pl-s"]],[[0,54,"pl-s"]],[[0,56,"pl-s"]],[[0,58,"pl-s"]],[[0,0,"pl-s"]],[[0,6,"pl-s"]],[[0,0,"pl-s"]],[[0,54,"pl-s"]],[[0,49,"pl-s"]],[[0,36,"pl-s"]],[[0,50,"pl-s"]],[[0,36,"pl-s"]],[[0,48,"pl-s"]],[[0,0,"pl-s"]],[[0,296,"pl-s"]],[[0,0,"pl-s"]],[[0,46,"pl-s"]],[[0,0,"pl-s"]],[[0,83,"pl-s"]],[[0,0,"pl-s"]],[[0,166,"pl-s"]],[[0,0,"pl-s"]],[[0,47,"pl-s"]],[[0,0,"pl-s"]],[[0,110,"pl-s"]],[[0,0,"pl-s"]],[[0,361,"pl-s"]],[[0,0,"pl-s"]],[[0,117,"pl-s"]],[[0,0,"pl-s"]],[[0,253,"pl-s"]],[[0,0,"pl-s"]],[[0,26,"pl-s"]],[[0,0,"pl-s"]],[[0,146,"pl-s"]],[[0,0,"pl-s"]],[[0,355,"pl-s"]],[[0,0,"pl-s"]],[[0,41,"pl-s"]],[[0,0,"pl-s"]],[[0,375,"pl-s"]],[[0,0,"pl-s"]],[[0,352,"pl-s"]],[[0,0,"pl-s"]],[[0,77,"pl-s"]],[[0,0,"pl-s"]],[[0,21,"pl-s"]],[[0,0,"pl-s"]],[[0,59,"pl-s"]],[[0,0,"pl-s"]],[[0,137,"pl-s"],[136,137,"pl-pds"],[137,401,"pl-s"]],[],[[0,25,"pl-ent"]],[],[[0,48,"pl-s"]],[[0,50,"pl-s"]],[[0,26,"pl-s"]],[[0,24,"pl-s"]],[[0,63,"pl-s"]],[],[[0,165,"pl-s"]],[],[[0,18,"pl-c"],[0,1,"pl-c"]],[],[[0,32,"pl-s"]],[],[[0,46,"pl-s"]],[],[[0,426,"pl-s"]],[],[[0,42,"pl-s"]],[],[[0,48,"pl-s"]],[],[[0,59,"pl-ent"],[61,316,"pl-s"]],[],[[0,30,"pl-s"]],[],[[0,35,"pl-s"]]],"colorizedLines":null}},"title":"phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide","appPayload":{},"meta":{"title":"phantomtide/docs/oom-postmortem.md at main · tg12/phantomtide"}}</script>
|
||
<div data-target="react-app.reactRoot"><meta name="github-code-view-meta-stats" id="github-code-view-meta-stats" data-hydrostats="publish"/> <!-- --> <a hidden="" id="code-view-repo-link" href="/tg12/phantomtide" data-discover="true"></a> <div class="d-none"></div><div><div style="--spacing:var(--spacing-none)" class="prc-PageLayout-PageLayoutRoot--KH-d" data-component="SplitPageLayout" data-has-sidebar="true"><div class="prc-PageLayout-SidebarWrapper-kLG4B CopilotSidePanelSidebar-module__SidePanel__L3O0C CopilotSidePanelSidebar-module__HiddenSidePanel__TBRGn" style="--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="end" data-sticky="true" data-responsive-variant="fullscreen"><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-SidebarVerticalDivider-0Rl0V" data-component="PageLayout.VerticalDivider" data-variant="line" data-position="end" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="450" aria-valuemax="768" aria-valuenow="544" aria-valuetext="Pane width 544 pixels" tabindex="0"></div></div><div class="prc-PageLayout-Sidebar-iciWg" data-component="SplitPageLayout.Sidebar" data-resizable="true" style="--spacing:var(--spacing-normal);--pane-min-width:450px;--pane-max-width:768px;--pane-width-custom:544px;--pane-width-size:var(--pane-width-custom);--pane-width:544px"><div class="height-full" data-testid="copilot-code-view-side-panel"><div id="copilot-side-panel-content" class="height-full"></div></div></div></div><div class="prc-PageLayout-PageLayoutWrapper-2BhU2" data-width="full"><div class="prc-PageLayout-PageLayoutContent-BneH9"><div id="repos-file-tree-sidebar" class="CodeViewFileTreeLayout-module__sidebar__n_Aau" tabindex="0"><div class="prc-PageLayout-PaneWrapper-pHPop ReposFileTreePane-module__Pane__rBZpI ReposFileTreePane-module__HideTree__AYZnm ReposFileTreePane-module__HidePane__VHAVt" style="--offset-header:0px;--spacing-row:var(--spacing-none);--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="start" data-sticky="true"><div class="prc-PageLayout-HorizontalDivider-JLVqp prc-PageLayout-PaneHorizontalDivider-9tbnE" data-component="PageLayout.HorizontalDivider" data-variant-regular="none" data-variant-narrow="none" data-position="start" style="--spacing-divider:var(--spacing-none);--spacing:var(--spacing-none)"></div><div class="prc-PageLayout-Pane-AyzHK" data-component="SplitPageLayout.Pane" data-resizable="true" style="--spacing:var(--spacing-none);--pane-min-width:256px;--pane-max-width:calc(100vw - var(--pane-max-width-diff));--pane-width-size:var(--pane-width-large);--pane-width:320px"></div><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-PaneVerticalDivider-le57g" data-component="PageLayout.VerticalDivider" data-variant-narrow="none" data-variant-regular="line" data-variant-wide="line" data-position="start" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="256" aria-valuemax="600" aria-valuenow="320" aria-valuetext="Pane width 320 pixels" tabindex="0"></div></div></div></div><div data-component="SplitPageLayout.Content" class="prc-PageLayout-ContentWrapper-gR9eG"><div class="prc-PageLayout-Content-xWL-A" data-width="full" style="--spacing:var(--spacing-none)"><div class="SharedPageLayout-module__content__IwGAp" data-selector="repos-split-pane-content" id="repos-split-pane-content" tabindex="0"> <!-- --> <div class="container CodeViewHeader-module__Box__JkPOb"><div class="CodeViewHeader-module__StickyHeader__Qn7UN" id="StickyHeader"><div class="CodeViewHeader-module__Box_1__SbNDV"><div class="CodeViewHeader-module__Box_2__TB46f"><div class="react-code-view-header-wrap--narrow CodeViewHeader-module__Box_3__q1zUL"><div class="CodeViewHeader-module__treeToggleWrapper__RQ__9"><h2 class="use-tree-pane-module__Heading__s4QbZ prc-Heading-Heading-MtWFE" data-component="Heading"><button data-component="Button" type="button" aria-label="Expand file tree" data-testid="expand-file-tree-button-mobile" class="prc-Button-ButtonBase-9n-Xk ExpandFileTreeButton-module__Button_1__Svs95" data-loading="false" data-size="medium" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-left" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.78 12.53a.75.75 0 0 1-1.06 0L2.47 8.28a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L4.81 7h7.44a.75.75 0 0 1 0 1.5H4.81l2.97 2.97a.75.75 0 0 1 0 1.06Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Files</span></span></button><button data-component="IconButton" type="button" data-testid="expand-file-tree-button" aria-controls="repos-file-tree" class="prc-Button-ButtonBase-9n-Xk position-relative ExpandFileTreeButton-module__expandButton__hDOcv ExpandFileTreeButton-module__filesButtonBreakpoint__zEvz3 fgColor-muted prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-labelledby="_R_4lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sidebar-collapse" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.823 7.823a.25.25 0 0 1 0 .354l-2.396 2.396A.25.25 0 0 1 4 10.396V5.604a.25.25 0 0 1 .427-.177Z"></path><path d="M1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0ZM1.5 1.75v12.5c0 .138.112.25.25.25H9.5v-13H1.75a.25.25 0 0 0-.25.25ZM11 14.5h3.25a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25H11Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="se" data-component="Tooltip" aria-hidden="true" id="_R_4lla9lik5_">Expand file tree</span><div class="d-none"></div></h2></div><div class="react-code-view-header-mb--narrow mr-2"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="main branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector-wide"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">main</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="react-code-view-header-mb--narrow CodeViewHeader-module__Box_5__MQ0hL"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__lg__Rjz0A"><nav data-testid="breadcrumbs" aria-labelledby="repos-header-breadcrumb-heading" id="repos-header-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="repos-header-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/tg12/phantomtide/tree/main" data-discover="true">phantomtide</a></li><li class="Breadcrumb-module__listItem__Ib0x_"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__lg__Rjz0A" aria-hidden="true">/</span><a class="Breadcrumb-module__directoryLink__kQy_t prc-Link-Link-9ZwDx" data-component="Link" href="/tg12/phantomtide/tree/main/docs" data-discover="true">docs</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__lg__Rjz0A" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__lg__Rjz0A prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="file-name-id">oom-postmortem.md</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lla9lik5_">Copy path</span></div></div></div><div class="react-code-view-header-element--wide"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><div><div class="CodeViewHeader-module__FileResultsList__JDzUy"><span class="d-flex FileResultsList-module__FilesSearchBox__ivVkc TextInput-wrapper prc-components-TextInputWrapper-Hpdqi prc-components-TextInputBaseWrapper-wY-n0" data-no-trailing-action="true" data-component="TextInput" data-leading-visual="true" data-trailing-visual="true" aria-busy="false"><span class="TextInput-icon" id="_R_5ipla9lik5_" aria-hidden="true" data-component="TextInput.LeadingVisual"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg></span><input type="text" aria-label="Go to file" role="combobox" aria-controls="file-results-list" aria-expanded="false" aria-haspopup="dialog" autoCorrect="off" spellCheck="false" placeholder="Go to file" aria-describedby="_R_5ipla9lik5_ _R_5ipla9lik5H1_" data-component="input" class="prc-components-Input-IwWrt" value=""/><span class="TextInput-icon" id="_R_5ipla9lik5H1_" aria-hidden="true" data-component="TextInput.TrailingVisual"></span></span></div><div class="d-none"></div></div><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-wide" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4pla9lik5_" id="_R_14pla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4pla9lik5_">More file actions</span></div></div></div><div class="react-code-view-header-element--narrow"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-narrow" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4tla9lik5_" id="_R_14tla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4tla9lik5_">More file actions</span></div></div></div></div></div></div></div><div class="CodeView-module__contentWrapper__cG2JH"><div class="react-code-view-bottom-padding"><div class="BlobTopBanners-module__Box__v_nvx"></div></div> <div class="d-none"></div><div class="d-flex flex-column border rounded-2 tmp-mb-3 pl-1"><div class="LatestCommit-module__Box__B25ZT"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">Latest commit</h2><div style="width:120px" class="Skeleton Skeleton--text" data-testid="loading"> </div><div class="d-flex flex-shrink-0 gap-2"><div data-testid="latest-commit-details" class="d-none d-sm-flex flex-items-center"></div><div class="d-flex gap-2"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">History</h2><a data-component="LinkButton" href="/tg12/phantomtide/commits/main/docs/oom-postmortem.md" class="prc-Button-ButtonBase-9n-Xk d-none d-lg-flex LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><span class="fgColor-default">History</span></span></span></a><div class="d-sm-none"></div><div class="d-flex d-lg-none"><a data-component="LinkButton" aria-label="View commit history for this file." href="/tg12/phantomtide/commits/main/docs/oom-postmortem.md" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby="_R_4mlala9lik5_"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span></span></a><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" role="tooltip" aria-hidden="true" id="_R_4mlala9lik5_">History</span></div></div></div></div></div><div class="d-flex flex-row"><div class="container BlobViewContent-module__blobContainer__DtH2d"><div class="react-code-size-details-banner BlobViewContent-module__codeSizeDetails__e5sUw"><div class="react-code-size-details-banner CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="20.5 KB" style="--truncate-max-width:100%"><span>350 lines (217 loc) · 20.5 KB</span></div></div></div></div><div class="react-blob-view-header-sticky BlobViewContent-module__stickyHeader__VwxB5" id="repos-sticky-header"><div class="BlobViewHeader-module__Box__yhm9u"><div class="react-blob-sticky-header"><div class="FileNameStickyHeader-module__outerWrapper__ZL4Xc FileNameStickyHeader-module__outerWrapperHidden__Zpynk"><div class="FileNameStickyHeader-module__Box_1__Hazu5"><div class="FileNameStickyHeader-module__Box_2__hoolP"><div class="FileNameStickyHeader-module__Box_3__MVKsk"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="main branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">main</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="FileNameStickyHeader-module__Box_4__FLhtt"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__md__Wb1Gs"><nav data-testid="breadcrumbs" aria-labelledby="sticky-breadcrumb-heading" id="sticky-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="sticky-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/tg12/phantomtide/tree/main" data-discover="true">phantomtide</a></li><li class="Breadcrumb-module__listItem__Ib0x_"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__md__Wb1Gs" aria-hidden="true">/</span><a class="Breadcrumb-module__directoryLink__kQy_t prc-Link-Link-9ZwDx" data-component="Link" href="/tg12/phantomtide/tree/main/docs" data-discover="true">docs</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__md__Wb1Gs" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__md__Wb1Gs prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="sticky-file-name-id">oom-postmortem.md</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lcpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lcpala9lik5_">Copy path</span></div></div></div><button data-component="Button" type="button" class="prc-Button-ButtonBase-9n-Xk FileNameStickyHeader-module__Button__LSEU_ FileNameStickyHeader-module__GoToTopButton__nxAFn" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-up" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.47 7.78a.75.75 0 0 1 0-1.06l4.25-4.25a.75.75 0 0 1 1.06 0l4.25 4.25a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018L9 4.81v7.44a.75.75 0 0 1-1.5 0V4.81L4.53 7.78a.75.75 0 0 1-1.06 0Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Top</span></span></button></div></div></div><div class="BlobViewHeader-module__Box_1__VEmuQ"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">File metadata and controls</h2><div class="BlobViewHeader-module__Box_2__icUs2"><ul aria-label="File view" class="prc-SegmentedControl-SegmentedControl-lqIXp BlobTabButtons-module__SegmentedControl__jen2u" data-variant="default" data-size="small" data-component="SegmentedControl"><li class="prc-SegmentedControl-Item-tSCQh" data-selected="" data-component="SegmentedControl.Button"><button aria-pressed="true" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:transparent"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Preview">Preview</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Code">Code</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Blame">Blame</div></span></button></li></ul><div class="d-none"></div><div class="react-code-size-details-in-header CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="20.5 KB" style="--truncate-max-width:100%"><span>350 lines (217 loc) · 20.5 KB</span></div></div></div></div><div class="BlobViewHeader-module__Box_3__ng6v2"><div class="d-none"></div><div class="react-blob-header-edit-and-raw-actions BlobViewHeader-module__Box_4__J4Y4W"><div class="d-none"></div><div class="prc-ButtonGroup-ButtonGroup-vFUrY" data-component="ButtonGroup"><div class="prc-ButtonGroup-Item-PqvDl"><a data-component="LinkButton" href="https://github.com/tg12/phantomtide/raw/refs/heads/main/docs/oom-postmortem.md" data-testid="raw-button" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov BlobViewHeader-module__LinkButton__X9kx2" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Raw</span></span></a></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="copy-raw-button" class="prc-Button-ButtonBase-9n-Xk prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_qaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_qaucpala9lik5_">Copy raw file</span></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="download-raw-button" class="prc-Button-ButtonBase-9n-Xk BlobViewHeader-module__downloadButton__ef459 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_eaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-download" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2.75 14A1.75 1.75 0 0 1 1 12.25v-2.5a.75.75 0 0 1 1.5 0v2.5c0 .138.112.25.25.25h10.5a.25.25 0 0 0 .25-.25v-2.5a.75.75 0 0 1 1.5 0v2.5A1.75 1.75 0 0 1 13.25 14Z"></path><path d="M7.25 7.689V2a.75.75 0 0 1 1.5 0v5.689l1.97-1.969a.749.749 0 1 1 1.06 1.06l-3.25 3.25a.749.749 0 0 1-1.06 0L4.22 6.78a.749.749 0 1 1 1.06-1.06l1.97 1.969Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_eaucpala9lik5_">Download raw file</span></div></div></div><button data-component="IconButton" type="button" aria-pressed="false" class="prc-Button-ButtonBase-9n-Xk tmp-mr-2 TableOfContents-module__IconButton__jrlNM prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_3ucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-list-unordered" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.75 2.5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2 14a1 1 0 1 1 0-2 1 1 0 0 1 0 2Zm1-6a1 1 0 1 1-2 0 1 1 0 0 1 2 0ZM2 4a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_3ucpala9lik5_">Outline</span><div class="react-blob-header-edit-and-raw-actions-combined"><button data-component="IconButton" type="button" title="More file actions" data-testid="more-file-actions-button" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click BlobViewHeader-module__IconButton__XrMQY prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_fkecpala9lik5_" id="_R_kecpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_fkecpala9lik5_">Edit and raw actions</span></div></div></div></div><div></div></div><div class="BlobViewContent-module__blobContentWrapper__JS0W6"><section aria-labelledby="file-name-id-wide file-name-id-mobile" class="BlobContent-module__blobContentSection__VOgZq BlobContent-module__blobContentSectionMarkdown__mPLOK" style="margin-top:46px"><div class="js-snippet-clipboard-copy-unpositioned BlobContent-module__markdownBlob__T8jpG" data-hpc="true" containertiming="hpc"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">How I Dragged Phantom Tide Out of an OOM Kill Loop</h1><a id="user-content-how-i-dragged-phantom-tide-out-of-an-oom-kill-loop" class="anchor" aria-label="Permalink: How I Dragged Phantom Tide Out of an OOM Kill Loop" href="#how-i-dragged-phantom-tide-out-of-an-oom-kill-loop"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">There is a certain kind of technical failure that tells you more about a platform than any polished launch thread ever will.</p>
|
||
<p dir="auto">Not when the UI looks clean. Not when the homepage loads fast. Not when the architecture diagram is neatly colour-coded and everyone is pretending the system is more deliberate than it really is. The real test is what happens when production turns hostile, memory gets tight, startup paths collide, caches swell, schedulers stampede, and the kernel starts killing your process like it has lost patience with your design.</p>
|
||
<p dir="auto">That was the point where Phantom Tide became real.</p>
|
||
<p dir="auto">From the outside, the symptom was boring: 502s across the site. From the inside, it was a systems failure spread across FastAPI, uvicorn, Redis, ClickHouse, APScheduler, Docker memory limits, and a startup sequence that had quietly become a deterministic self-attack.</p>
|
||
<p dir="auto">What follows is the technical story of how I fixed it. But it is also the real story of what Phantom Tide actually is.</p>
|
||
<p dir="auto">Because the uncomfortable truth is this: I did not build a map.</p>
|
||
<p dir="auto">The map is just the visible surface. The real work was underneath it: source ingestion, event normalisation, scheduler behaviour, memory control, failure isolation, storage trade-offs, and the ugly operational logic required to make hostile upstream data behave like one coherent system. If you only look at Phantom Tide as a frontend product, you miss the engineering entirely. The map is just the proof that the machinery resolves into something legible. The machinery is the point.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">I thought I was fixing a crash loop. I was really exposing the architecture.</h2><a id="user-content-i-thought-i-was-fixing-a-crash-loop-i-was-really-exposing-the-architecture" class="anchor" aria-label="Permalink: I thought I was fixing a crash loop. I was really exposing the architecture." href="#i-thought-i-was-fixing-a-crash-loop-i-was-really-exposing-the-architecture"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">At first glance this looked like an ordinary outage. Phantom Tide started returning 502 Bad Gateway responses and the application container was restart-looping. That is the sort of thing a lot of people write off as “server trouble” until they burn hours chasing reverse proxies and restart counters.</p>
|
||
<p dir="auto">I ignored the polite lies and went straight to the kernel.</p>
|
||
<p dir="auto"><code>dmesg</code> made it clear almost immediately that this was not a routing issue or a generic application fault. The Python process was being killed by the OOM killer. The key signal was <code>anon-rss</code>, not total VM size, not Docker restart metadata, not container health status. Actual resident anonymous memory was blowing through the limit and the kernel was enforcing it.</p>
|
||
<p dir="auto">That matters because once the kernel is killing your process, the question stops being “why is the site down” and becomes “which part of my system thinks it deserves this much memory, and why did I let it.”</p>
|
||
<p dir="auto">That is the difference between debugging the symptom and debugging the system.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The first problem: I was treating a stateful process like a stateless web app</h2><a id="user-content-the-first-problem-i-was-treating-a-stateful-process-like-a-stateless-web-app" class="anchor" aria-label="Permalink: The first problem: I was treating a stateful process like a stateless web app" href="#the-first-problem-i-was-treating-a-stateful-process-like-a-stateless-web-app"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">The initial deployment was running uvicorn with four workers.</p>
|
||
<p dir="auto">That is normal if your application is basically stateless request handling and you want cheap concurrency. It is a bad idea when your process contains mutable in-memory state, a scheduler, startup hydration logic, live buffers, and background collectors that assume singleton behaviour.</p>
|
||
<p dir="auto">In that setup, four workers do not mean one application serving four times more efficiently. It means four copies of the same process state, four schedulers, four startup passes, four independent memory footprints, and four ways to multiply every mistake hidden in your boot logic.</p>
|
||
<p dir="auto">That is not resilience. That is replication of instability.</p>
|
||
<p dir="auto">So the first emergency change was brutally simple:</p>
|
||
<div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="uvicorn phantom_app.main:app --host 0.0.0.0 --port 8000 --workers 1"><pre>uvicorn phantom_app.main:app --host 0.0.0.0 --port 8000 --workers 1</pre></div>
|
||
<p dir="auto">That did not solve the root cause. What it did was stop the platform from amplifying it. Sometimes the first real fix is not elegant. It is just removing the part that is making everything else harder to see.</p>
|
||
<p dir="auto">This was also the first reminder of what Phantom Tide actually proves. Not that I can wire together a few services and make a dashboard. That I understand when architectural defaults stop being defaults and start becoming liabilities.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The second problem: 28 MB in Redis became roughly 2 GB in Python</h2><a id="user-content-the-second-problem-28-mb-in-redis-became-roughly-2-gb-in-python" class="anchor" aria-label="Permalink: The second problem: 28 MB in Redis became roughly 2 GB in Python" href="#the-second-problem-28-mb-in-redis-became-roughly-2-gb-in-python"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Once the worker count dropped, the next obvious suspect was Redis.</p>
|
||
<p dir="auto">The live event set contained around 20,000 records. The backing JSON footprint was roughly 28 MB. That sounds harmless if you think in terms of storage size. It is not harmless if you understand what Python does to structured data once you fully materialise it into live objects.</p>
|
||
<p dir="auto">On startup, the app was hydrating the event store by deserialising every record from the Redis sorted set and constructing dataclass instances for each one. Each <code>EventRecord</code> carried around 22 fields plus nested structures such as attributes, geometry, and raw source data. That may look compact in JSON. It is not compact in memory.</p>
|
||
<p dir="auto">JSON is just bytes. Python objects are a managed heap full of per-object overhead, instance dictionaries, nested dictionaries, strings, references, allocator fragmentation, and garbage collector metadata. So a few dozen megabytes of stored data turned into a giant in-process object graph.</p>
|
||
<p dir="auto">The result was approximately a 70x inflation from Redis JSON size to live heap use.</p>
|
||
<p dir="auto">This is exactly the sort of thing that never shows up in superficial product writeups. You can talk all day about “real-time intelligence fusion,” but eventually the truth comes down to ugly details like whether your startup path is quietly reconstructing two gigabytes of Python objects because you were lazy about hydration strategy.</p>
|
||
<p dir="auto">That was the next fix.</p>
|
||
<p dir="auto">First I reduced retention. Then I enforced the limit at hydration time. Then I stopped eagerly materialising the entire backlog as one big in-memory list.</p>
|
||
<p dir="auto">The retention and load limits became explicit:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="MAX_REDIS_EVENTS = 8_000
|
||
redis_store.load_recent_events(hours=24, limit=8_000)
|
||
ch_store.load_recent_events(hours=24, limit=8_000)"><pre><span class="pl-c1">MAX_REDIS_EVENTS</span> <span class="pl-c1">=</span> <span class="pl-c1">8_000</span>
|
||
<span class="pl-s1">redis_store</span>.<span class="pl-c1">load_recent_events</span>(<span class="pl-s1">hours</span><span class="pl-c1">=</span><span class="pl-c1">24</span>, <span class="pl-s1">limit</span><span class="pl-c1">=</span><span class="pl-c1">8_000</span>)
|
||
<span class="pl-s1">ch_store</span>.<span class="pl-c1">load_recent_events</span>(<span class="pl-s1">hours</span><span class="pl-c1">=</span><span class="pl-c1">24</span>, <span class="pl-s1">limit</span><span class="pl-c1">=</span><span class="pl-c1">8_000</span>)</pre></div>
|
||
<p dir="auto">Live Redis was trimmed immediately so the next restart would not drag the old problem straight back in:</p>
|
||
<div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="docker exec phantom_redis redis-cli ZREMRANGEBYRANK phantom:events 0 -8001"><pre>docker <span class="pl-c1">exec</span> phantom_redis redis-cli ZREMRANGEBYRANK phantom:events 0 -8001</pre></div>
|
||
<p dir="auto">Then I changed the hydration path so it streamed records instead of building a giant list up front:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="def hydrate_from_redis_iter(redis_conn):
|
||
for _, raw in redis_conn.zscan_iter('phantom:events'):
|
||
yield EventRecord(**json.loads(raw))
|
||
|
||
def insert_streamed(client, it, chunk_size=1000):
|
||
buffer = []
|
||
for rec in it:
|
||
buffer.append(rec.to_tuple())
|
||
if len(buffer) >= chunk_size:
|
||
client.insert('events', buffer)
|
||
buffer.clear()
|
||
if buffer:
|
||
client.insert('events', buffer)"><pre><span class="pl-k">def</span> <span class="pl-en">hydrate_from_redis_iter</span>(<span class="pl-s1">redis_conn</span>):
|
||
<span class="pl-k">for</span> <span class="pl-s1">_</span>, <span class="pl-s1">raw</span> <span class="pl-c1">in</span> <span class="pl-s1">redis_conn</span>.<span class="pl-c1">zscan_iter</span>(<span class="pl-s">'phantom:events'</span>):
|
||
<span class="pl-k">yield</span> <span class="pl-en">EventRecord</span>(<span class="pl-c1">**</span><span class="pl-s1">json</span>.<span class="pl-c1">loads</span>(<span class="pl-s1">raw</span>))
|
||
|
||
<span class="pl-k">def</span> <span class="pl-en">insert_streamed</span>(<span class="pl-s1">client</span>, <span class="pl-s1">it</span>, <span class="pl-s1">chunk_size</span><span class="pl-c1">=</span><span class="pl-c1">1000</span>):
|
||
<span class="pl-s1">buffer</span> <span class="pl-c1">=</span> []
|
||
<span class="pl-k">for</span> <span class="pl-s1">rec</span> <span class="pl-c1">in</span> <span class="pl-s1">it</span>:
|
||
<span class="pl-s1">buffer</span>.<span class="pl-c1">append</span>(<span class="pl-s1">rec</span>.<span class="pl-c1">to_tuple</span>())
|
||
<span class="pl-k">if</span> <span class="pl-en">len</span>(<span class="pl-s1">buffer</span>) <span class="pl-c1">>=</span> <span class="pl-s1">chunk_size</span>:
|
||
<span class="pl-s1">client</span>.<span class="pl-c1">insert</span>(<span class="pl-s">'events'</span>, <span class="pl-s1">buffer</span>)
|
||
<span class="pl-s1">buffer</span>.<span class="pl-c1">clear</span>()
|
||
<span class="pl-k">if</span> <span class="pl-s1">buffer</span>:
|
||
<span class="pl-s1">client</span>.<span class="pl-c1">insert</span>(<span class="pl-s">'events'</span>, <span class="pl-s1">buffer</span>)</pre></div>
|
||
<p dir="auto">I also reduced per-object overhead on the hot path with slots-based dataclasses:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="@dataclass(slots=True)
|
||
class EventRecord:
|
||
..."><pre><span class="pl-en">@<span class="pl-en">dataclass</span>(<span class="pl-s1">slots</span><span class="pl-c1">=</span><span class="pl-c1">True</span>)</span>
|
||
<span class="pl-k">class</span> <span class="pl-v">EventRecord</span>:
|
||
...</pre></div>
|
||
<p dir="auto">That did not magically make Python cheap. It did make the startup path stop behaving like a memory ambush. Staging memray runs showed a material drop in peak usage on the hydration path after streaming and <code>slots=True</code>.</p>
|
||
<p dir="auto">Again, this is the part most people bury because it is not glamorous. I think it is the most convincing part of the whole project. The engineering value in Phantom Tide is not that it displays events on a map. It is that it takes badly behaved, heterogeneous upstream data and forces it through a system that can be reasoned about, profiled, and corrected under production pressure.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The third problem: the scheduler was launching a coordinated attack on startup</h2><a id="user-content-the-third-problem-the-scheduler-was-launching-a-coordinated-attack-on-startup" class="anchor" aria-label="Permalink: The third problem: the scheduler was launching a coordinated attack on startup" href="#the-third-problem-the-scheduler-was-launching-a-coordinated-attack-on-startup"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">After reducing worker multiplication and fixing the worst of the hydration path, the platform still died.</p>
|
||
<p dir="auto">The important clue was timing. The app would boot, sit at a few hundred megabytes, look stable, and then die exactly thirty seconds later.</p>
|
||
<p dir="auto">Not roughly. Exactly.</p>
|
||
<p dir="auto">That told me this was not a vague leak or gradual accumulation problem. It was a deterministic startup event.</p>
|
||
<p dir="auto">The culprit was APScheduler.</p>
|
||
<p dir="auto">To ensure the platform had fresh data as soon as it started, the scheduler registered one-shot startup jobs for the slower collectors before their normal intervals began. That part was fine. The bad part was that all of them were assigned the same <code>next_run_time</code>.</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="startup_run_time = datetime.now() + timedelta(seconds=30)
|
||
|
||
for key, fn, _interval, job_id, job_name, startup_run in slow_jobs:
|
||
collector = popped.get(key)
|
||
if collector is not None and startup_run:
|
||
_scheduler.add_job(
|
||
fn,
|
||
args=[collector],
|
||
id=f"collectors_{job_id}_startup",
|
||
name=f"Initial {job_name} run",
|
||
next_run_time=startup_run_time,
|
||
)"><pre><span class="pl-s1">startup_run_time</span> <span class="pl-c1">=</span> <span class="pl-s1">datetime</span>.<span class="pl-c1">now</span>() <span class="pl-c1">+</span> <span class="pl-en">timedelta</span>(<span class="pl-s1">seconds</span><span class="pl-c1">=</span><span class="pl-c1">30</span>)
|
||
|
||
<span class="pl-k">for</span> <span class="pl-s1">key</span>, <span class="pl-s1">fn</span>, <span class="pl-s1">_interval</span>, <span class="pl-s1">job_id</span>, <span class="pl-s1">job_name</span>, <span class="pl-s1">startup_run</span> <span class="pl-c1">in</span> <span class="pl-s1">slow_jobs</span>:
|
||
<span class="pl-s1">collector</span> <span class="pl-c1">=</span> <span class="pl-s1">popped</span>.<span class="pl-c1">get</span>(<span class="pl-s1">key</span>)
|
||
<span class="pl-k">if</span> <span class="pl-s1">collector</span> <span class="pl-c1"><span class="pl-c1">is</span> <span class="pl-c1">not</span></span> <span class="pl-c1">None</span> <span class="pl-c1">and</span> <span class="pl-s1">startup_run</span>:
|
||
<span class="pl-s1">_scheduler</span>.<span class="pl-c1">add_job</span>(
|
||
<span class="pl-s1">fn</span>,
|
||
<span class="pl-s1">args</span><span class="pl-c1">=</span>[<span class="pl-s1">collector</span>],
|
||
<span class="pl-s1">id</span><span class="pl-c1">=</span><span class="pl-s">f"collectors_<span class="pl-s1"><span class="pl-kos">{</span><span class="pl-s1">job_id</span><span class="pl-kos">}</span></span>_startup"</span>,
|
||
<span class="pl-s1">name</span><span class="pl-c1">=</span><span class="pl-s">f"Initial <span class="pl-s1"><span class="pl-kos">{</span><span class="pl-s1">job_name</span><span class="pl-kos">}</span></span> run"</span>,
|
||
<span class="pl-s1">next_run_time</span><span class="pl-c1">=</span><span class="pl-s1">startup_run_time</span>,
|
||
)</pre></div>
|
||
<p dir="auto">So at T+30 seconds, a whole wall of collectors hit the system simultaneously.</p>
|
||
<p dir="auto">NDBC. MARAD. ICC-CCS. GPS advisory sources. USGS. ECCC. VIIRS. NGA MIS. DART. Multiple network fetches, parsers, object allocations, and store writes all landing at once on a constrained VM.</p>
|
||
<p dir="auto">That is not startup. That is a deterministic burst load disguised as startup.</p>
|
||
<p dir="auto">This is where the broader product story matters. Phantom Tide is not just “aggregating feeds.” It is a distributed argument about reality across sources that disagree, drift, stall, overproduce, and break in different ways. The engineering challenge is not collecting the signals. It is making those signals survive contact with a live system. Startup scheduling is part of that. If your collectors all fire as one synchronized burst, you have built a fragility engine, not an intelligence platform.</p>
|
||
<p dir="auto">The fix was to introduce jitter and spread the one-shot jobs across a startup window:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="startup_delay = int(os.getenv("STARTUP_COLLECT_DELAY_SECONDS", "30"))
|
||
startup_window = int(os.getenv("STARTUP_COLLECT_WINDOW_SECONDS", "180"))
|
||
|
||
def _staggered_time(base_delay: int, window: int) -> datetime:
|
||
offset = random.randint(0, max(0, window))
|
||
return datetime.now() + timedelta(seconds=base_delay + offset)
|
||
|
||
for key, fn, _interval, job_id, job_name, startup_run in slow_jobs:
|
||
collector = popped.get(key)
|
||
if collector is not None and startup_run:
|
||
_scheduler.add_job(
|
||
fn,
|
||
args=[collector],
|
||
id=f"collectors_{job_id}_startup",
|
||
name=f"Initial {job_name} run",
|
||
next_run_time=_staggered_time(startup_delay, startup_window),
|
||
)"><pre><span class="pl-s1">startup_delay</span> <span class="pl-c1">=</span> <span class="pl-en">int</span>(<span class="pl-s1">os</span>.<span class="pl-c1">getenv</span>(<span class="pl-s">"STARTUP_COLLECT_DELAY_SECONDS"</span>, <span class="pl-s">"30"</span>))
|
||
<span class="pl-s1">startup_window</span> <span class="pl-c1">=</span> <span class="pl-en">int</span>(<span class="pl-s1">os</span>.<span class="pl-c1">getenv</span>(<span class="pl-s">"STARTUP_COLLECT_WINDOW_SECONDS"</span>, <span class="pl-s">"180"</span>))
|
||
|
||
<span class="pl-k">def</span> <span class="pl-en">_staggered_time</span>(<span class="pl-s1">base_delay</span>: <span class="pl-smi">int</span>, <span class="pl-s1">window</span>: <span class="pl-smi">int</span>) <span class="pl-c1">-></span> <span class="pl-smi">datetime</span>:
|
||
<span class="pl-s1">offset</span> <span class="pl-c1">=</span> <span class="pl-s1">random</span>.<span class="pl-c1">randint</span>(<span class="pl-c1">0</span>, <span class="pl-en">max</span>(<span class="pl-c1">0</span>, <span class="pl-s1">window</span>))
|
||
<span class="pl-k">return</span> <span class="pl-s1">datetime</span>.<span class="pl-c1">now</span>() <span class="pl-c1">+</span> <span class="pl-en">timedelta</span>(<span class="pl-s1">seconds</span><span class="pl-c1">=</span><span class="pl-s1">base_delay</span> <span class="pl-c1">+</span> <span class="pl-s1">offset</span>)
|
||
|
||
<span class="pl-k">for</span> <span class="pl-s1">key</span>, <span class="pl-s1">fn</span>, <span class="pl-s1">_interval</span>, <span class="pl-s1">job_id</span>, <span class="pl-s1">job_name</span>, <span class="pl-s1">startup_run</span> <span class="pl-c1">in</span> <span class="pl-s1">slow_jobs</span>:
|
||
<span class="pl-s1">collector</span> <span class="pl-c1">=</span> <span class="pl-s1">popped</span>.<span class="pl-c1">get</span>(<span class="pl-s1">key</span>)
|
||
<span class="pl-k">if</span> <span class="pl-s1">collector</span> <span class="pl-c1"><span class="pl-c1">is</span> <span class="pl-c1">not</span></span> <span class="pl-c1">None</span> <span class="pl-c1">and</span> <span class="pl-s1">startup_run</span>:
|
||
<span class="pl-s1">_scheduler</span>.<span class="pl-c1">add_job</span>(
|
||
<span class="pl-s1">fn</span>,
|
||
<span class="pl-s1">args</span><span class="pl-c1">=</span>[<span class="pl-s1">collector</span>],
|
||
<span class="pl-s1">id</span><span class="pl-c1">=</span><span class="pl-s">f"collectors_<span class="pl-s1"><span class="pl-kos">{</span><span class="pl-s1">job_id</span><span class="pl-kos">}</span></span>_startup"</span>,
|
||
<span class="pl-s1">name</span><span class="pl-c1">=</span><span class="pl-s">f"Initial <span class="pl-s1"><span class="pl-kos">{</span><span class="pl-s1">job_name</span><span class="pl-kos">}</span></span> run"</span>,
|
||
<span class="pl-s1">next_run_time</span><span class="pl-c1">=</span><span class="pl-en">_staggered_time</span>(<span class="pl-s1">startup_delay</span>, <span class="pl-s1">startup_window</span>),
|
||
)</pre></div>
|
||
<p dir="auto">That turned the startup profile from a cliff into a slope. Instead of thirteen collectors hitting together, they spread out over a few minutes. Memory rose gradually. The platform stayed up.</p>
|
||
<p dir="auto">This was one of the most important fixes because it exposed a deeper truth: most real-world system failures are not a result of one dramatic bug. They come from individually reasonable choices interacting at exactly the wrong point in time.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The fourth problem: the fast loop was still colliding with the slow startup window</h2><a id="user-content-the-fourth-problem-the-fast-loop-was-still-colliding-with-the-slow-startup-window" class="anchor" aria-label="Permalink: The fourth problem: the fast loop was still colliding with the slow startup window" href="#the-fourth-problem-the-fast-loop-was-still-colliding-with-the-slow-startup-window"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">The first scheduler fix removed the mass simultaneous burst. It did not fully remove startup overlap.</p>
|
||
<p dir="auto">The fast-loop collectors still had their own initial run scheduled too early, which meant they could still collide with slow collectors that happened to land near the front of the stagger window.</p>
|
||
<p dir="auto">So the startup sequence needed a second pass. The fast loop had to move to the end of the slow-collector window instead of overlapping with it. That changed startup from “partially staggered chaos” into an actual controlled sequence.</p>
|
||
<p dir="auto">The rough shape became:</p>
|
||
<ul dir="auto">
|
||
<li>application boots</li>
|
||
<li>scheduler starts</li>
|
||
<li>slow collectors begin after the base delay, staggered across the window</li>
|
||
<li>only after that window closes does the fast-loop initial run fire</li>
|
||
</ul>
|
||
<p dir="auto">That sounds small. It was not small. It removed another artificial pressure point and made startup behave like something designed, not something improvised.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The fifth problem: ClickHouse was configured like it owned the box</h2><a id="user-content-the-fifth-problem-clickhouse-was-configured-like-it-owned-the-box" class="anchor" aria-label="Permalink: The fifth problem: ClickHouse was configured like it owned the box" href="#the-fifth-problem-clickhouse-was-configured-like-it-owned-the-box"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">At the same time, ClickHouse was competing for memory on the same host.</p>
|
||
<p dir="auto">That mattered because the box was shared. Phantom Tide was not running in isolation. Redis, ClickHouse, the app container, and supporting services were all drawing from the same physical machine. ClickHouse 26.2 also has opinions about startup allocations and where certain memory-related settings actually belong.</p>
|
||
<p dir="auto">This turned into a small piece of config archaeology. Some settings were overly aggressive for the environment. Some belonged in different config scopes. Some combinations created ugly startup behaviour. One XML mistake introduced exactly the sort of pointless delay you deserve when editing config under stress.</p>
|
||
<p dir="auto">The path out was iterative and boring in the right way:</p>
|
||
<ul dir="auto">
|
||
<li>reduce cache sizes</li>
|
||
<li>validate XML</li>
|
||
<li>strip back custom config</li>
|
||
<li>reintroduce only what is proven safe</li>
|
||
<li>observe startup under cgroup limits</li>
|
||
<li>stop treating ClickHouse like the host exists purely for it</li>
|
||
</ul>
|
||
<p dir="auto">That mattered because the platform’s strongest engineering story is not “I know how to install ClickHouse.” It is “I know how to operate a mixed system where each component lies differently under stress, and I know how to back the whole thing down to something stable without guessing.”</p>
|
||
<p dir="auto">If you want to show engineering skill, this is where it lives. Not in saying you used ClickHouse. In showing you understand when ClickHouse becomes the wrong neighbour on a tight box unless you discipline it.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse</h2><a id="user-content-the-sixth-problem-huge-flushes-were-creating-secondary-memory-spikes-in-clickhouse" class="anchor" aria-label="Permalink: The sixth problem: huge flushes were creating secondary memory spikes in ClickHouse" href="#the-sixth-problem-huge-flushes-were-creating-secondary-memory-spikes-in-clickhouse"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">The in-memory store had another bad habit. It was flushing pending events to ClickHouse as one large batch.</p>
|
||
<p dir="auto">That meant after a noisy restart or backlog recovery, the first insert could be enormous. ClickHouse would then need to process that insert in memory before writing it out, which is exactly the kind of thing you do not want happening on a machine already recovering from pressure.</p>
|
||
<p dir="auto">Before, it effectively looked like this:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="ch_store.insert_events(batch)"><pre><span class="pl-s1">ch_store</span>.<span class="pl-c1">insert_events</span>(<span class="pl-s1">batch</span>)</pre></div>
|
||
<p dir="auto">That is fine until <code>batch</code> is not fine.</p>
|
||
<p dir="auto">So I chunked it:</p>
|
||
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="_CH_FLUSH_CHUNK = 2_000
|
||
|
||
for i in range(0, len(batch), _CH_FLUSH_CHUNK):
|
||
chunk = batch[i:i + _CH_FLUSH_CHUNK]
|
||
ch_store.insert_events(chunk)"><pre><span class="pl-s1">_CH_FLUSH_CHUNK</span> <span class="pl-c1">=</span> <span class="pl-c1">2_000</span>
|
||
|
||
<span class="pl-k">for</span> <span class="pl-s1">i</span> <span class="pl-c1">in</span> <span class="pl-en">range</span>(<span class="pl-c1">0</span>, <span class="pl-en">len</span>(<span class="pl-s1">batch</span>), <span class="pl-s1">_CH_FLUSH_CHUNK</span>):
|
||
<span class="pl-s1">chunk</span> <span class="pl-c1">=</span> <span class="pl-s1">batch</span>[<span class="pl-s1">i</span>:<span class="pl-s1">i</span> <span class="pl-c1">+</span> <span class="pl-s1">_CH_FLUSH_CHUNK</span>]
|
||
<span class="pl-s1">ch_store</span>.<span class="pl-c1">insert_events</span>(<span class="pl-s1">chunk</span>)</pre></div>
|
||
<p dir="auto">That bounded per-insert memory pressure and made recovery behaviour much more predictable.</p>
|
||
<p dir="auto">That kind of change matters more than people think. Predictability is one of the few real luxuries you can buy in production. Large single-shot flushes feel efficient until they are the thing that tips a struggling system over the edge.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The seventh problem: zero swap headroom made every spike fatal</h2><a id="user-content-the-seventh-problem-zero-swap-headroom-made-every-spike-fatal" class="anchor" aria-label="Permalink: The seventh problem: zero swap headroom made every spike fatal" href="#the-seventh-problem-zero-swap-headroom-made-every-spike-fatal"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">The container limits were also too sharp.</p>
|
||
<p dir="auto">The app had effectively no swap breathing room:</p>
|
||
<div class="highlight highlight-source-yaml notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="mem_limit: 2g
|
||
memswap_limit: 2g"><pre><span class="pl-ent">mem_limit</span>: <span class="pl-c1">2g</span>
|
||
<span class="pl-ent">memswap_limit</span>: <span class="pl-c1">2g</span></pre></div>
|
||
<p dir="auto">That means any transient spike above 2 GB becomes an immediate death sentence. For a service with a rough startup path and bursty collector behaviour, that is needlessly unforgiving.</p>
|
||
<p dir="auto">Once the real fixes were in place, I widened the operating margin:</p>
|
||
<div class="highlight highlight-source-yaml notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="mem_limit: 4g
|
||
memswap_limit: 5g"><pre><span class="pl-ent">mem_limit</span>: <span class="pl-c1">4g</span>
|
||
<span class="pl-ent">memswap_limit</span>: <span class="pl-c1">5g</span></pre></div>
|
||
<p dir="auto">This did not replace proper engineering. It complemented it. There is a difference between using headroom to hide a bad design and using headroom to stop temporary spikes from killing an otherwise stabilised service. This was the latter.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">What changed after the fixes</h2><a id="user-content-what-changed-after-the-fixes" class="anchor" aria-label="Permalink: What changed after the fixes" href="#what-changed-after-the-fixes"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">After the worker reduction, Redis cap, streamed hydration, scheduler staggering, fast-loop sequencing, ClickHouse insert chunking, conservative ClickHouse tuning, and memory headroom adjustments, the platform stopped behaving like a brittle prototype and started behaving like an actual system.</p>
|
||
<p dir="auto">The important thing was not just that it stopped crashing. It was how it stopped crashing.</p>
|
||
<p dir="auto">Before:</p>
|
||
<ul dir="auto">
|
||
<li>startup looked normal and then detonated</li>
|
||
<li>all collectors behaved like they had no awareness of each other</li>
|
||
<li>Redis hydration inflated into a giant Python heap</li>
|
||
<li>ClickHouse was competing too aggressively for memory</li>
|
||
<li>one large persistence flush could create another spike</li>
|
||
<li>the container limits gave no room for transient pressure</li>
|
||
</ul>
|
||
<p dir="auto">After:</p>
|
||
<ul dir="auto">
|
||
<li>startup memory climbed gradually instead of cliffing</li>
|
||
<li>collectors came online in a controlled sequence</li>
|
||
<li>hydration was bounded and streamed</li>
|
||
<li>ClickHouse stopped acting like it owned the host</li>
|
||
<li>flush behaviour became predictable</li>
|
||
<li>the app settled into a sane steady-state range</li>
|
||
</ul>
|
||
<p dir="auto">That is the real engineering story of Phantom Tide. Not that it fuses AIS, OpenSky, NOTAMs, GPS interference, advisories, marine feeds, and other operational sources into one view. The stronger claim is that it does that while surviving contact with production realities that kill weaker systems.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">What this incident proved about the product</h2><a id="user-content-what-this-incident-proved-about-the-product" class="anchor" aria-label="Permalink: What this incident proved about the product" href="#what-this-incident-proved-about-the-product"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">This outage ended up clarifying the product better than any launch copy could have.</p>
|
||
<p dir="auto">I had been tempted, like everyone is, to talk about Phantom Tide as a live map or an intelligence interface. That is the easy description. It is also the shallow one.</p>
|
||
<p dir="auto">The deeper and more defensible framing is this:</p>
|
||
<p dir="auto">Phantom Tide is not a dashboard. It is a systems engineering case study disguised as an intelligence platform.</p>
|
||
<p dir="auto">The difficult part is not rendering events. The difficult part is deciding what deserves to become an event in the first place, how long it should live, how it should be structured, how conflicting sources should coexist, how startup should behave, how persistence should fail safely, and how the platform should degrade under pressure without becoming fiction.</p>
|
||
<p dir="auto">That is what the OOM incident surfaced. The value is not the glossy layer. It is the logic and discipline beneath it.</p>
|
||
<p dir="auto">A year from now, the thing I would regret most is underselling that by talking too much about the output and not enough about the machinery. The map is useful because the platform behind it earned the right to produce it. That is the part worth showing.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The uncomfortable truth</h2><a id="user-content-the-uncomfortable-truth" class="anchor" aria-label="Permalink: The uncomfortable truth" href="#the-uncomfortable-truth"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">If I marketed Phantom Tide mainly as “a live map that merges these sources,” a lot of technically serious people would underestimate it instantly.</p>
|
||
<p dir="auto">They would assume the hard part was UI composition and data plumbing. They would file it mentally under dashboard-builder. They would miss the actual signal: scheduler design, source reconciliation, object inflation, memory profiling, batch persistence strategy, cache-vs-archive reasoning, and live operational debugging under constrained infrastructure.</p>
|
||
<p dir="auto">That is why I think the mess is the moat.</p>
|
||
<p dir="auto">Not the mess in the sense of sloppy engineering. The mess in the sense of the real problems you only meet when a system is no longer theoretical. Bad upstreams. Structural drift. startup stampedes. false health. silent data loss. stateful assumptions hiding inside supposedly simple services. Those are not embarrassing side notes. They are the proof of contact with reality.</p>
|
||
<p dir="auto">Anyone can make a pretty interface over commodity data. Very few people can explain why 28 MB in Redis became 2 GB in Python, why four workers made the system worse, why startup jitter mattered more than another CPU, why ClickHouse needed discipline, and why archival persistence was still the next priority even after the platform looked stable again.</p>
|
||
<p dir="auto">That is the engineering skill. That is what Phantom Tide really demonstrates.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">What still remains</h2><a id="user-content-what-still-remains" class="anchor" aria-label="Permalink: What still remains" href="#what-still-remains"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">Fixing the OOM loop did not mean the platform was finished.</p>
|
||
<p dir="auto">The highest-priority remaining issue is ClickHouse insert failure caused by malformed field types. At least one collector is emitting a <code>dict</code> where ClickHouse expects a string or numeric field, which poisons entire insert batches. The in-memory store and Redis make the platform look healthy in the short term, but the archive path is not trustworthy until there is schema validation at the boundary.</p>
|
||
<p dir="auto">That is the next real job:</p>
|
||
<ul dir="auto">
|
||
<li>validate per-event payloads before persistence</li>
|
||
<li>log the source and field causing schema failures</li>
|
||
<li>coerce where appropriate</li>
|
||
<li>reject where necessary</li>
|
||
<li>stop silent data loss from hiding behind a healthy-looking UI</li>
|
||
</ul>
|
||
<p dir="auto">There are other source-specific issues as well, but those are normal platform work. The insert validation gap is the big one because it attacks trust in the archive.</p>
|
||
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">The real lesson</h2><a id="user-content-the-real-lesson" class="anchor" aria-label="Permalink: The real lesson" href="#the-real-lesson"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
|
||
<p dir="auto">The kernel told the truth first.</p>
|
||
<p dir="auto">Everything else was just me catching up to it.</p>
|
||
<p dir="auto">This was not one bug. It was a layered systems failure where worker topology, object modelling, startup choreography, batch behaviour, database configuration, and memory limits all interacted in exactly the wrong way. The platform was not failing because of a single bad line of code. It was failing because multiple “reasonable” decisions had combined into a design that only looked solid until production put pressure on it.</p>
|
||
<p dir="auto">That is why I think this incident matters.</p>
|
||
<p dir="auto">Not because the site went down. Systems go down.</p>
|
||
<p dir="auto">It matters because it exposed what Phantom Tide actually is: not a map, not a feed aggregator, not a dressed-up frontend project, but a live operational system built on top of messy, high-friction data, where the hardest work is imposing structure, discipline, and survivability on things that do not want to behave.</p>
|
||
<p dir="auto">That is the engineering story.</p>
|
||
<p dir="auto">And that is the part worth selling.</p>
|
||
</article></div><div class="d-none"></div></section></div></div></div> </div> <!-- --> </div></div></div></div></div></div><div class="ScrollMarksContainer-module__scrollMarksContainer__Eu7uU" id="find-result-marks-container"></div><div class="d-none"></div><div class="d-none"></div></div> <!-- --> <!-- --> </div>
|
||
</react-app>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</turbo-frame>
|
||
|
||
</main>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<footer class="footer f6 color-fg-muted color-border-subtle tmp-pt-7 tmp-pb-6 p-responsive" role="contentinfo" >
|
||
<h2 class='sr-only'>Footer</h2>
|
||
|
||
|
||
|
||
|
||
<div class="d-flex flex-justify-center flex-items-center flex-column-reverse flex-lg-row flex-wrap flex-lg-nowrap">
|
||
<div class="d-flex flex-items-center flex-shrink-0 mx-2">
|
||
<a aria-label="GitHub Homepage" class="footer-octicon mr-2" href="https://github.com">
|
||
<svg aria-hidden="true" data-component="Octicon" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-mark-github">
|
||
<path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path>
|
||
</svg>
|
||
</a>
|
||
<span>
|
||
© 2026 GitHub, Inc.
|
||
</span>
|
||
</div>
|
||
|
||
<nav aria-label="Footer">
|
||
<h3 class="sr-only" id="sr-footer-heading">Footer navigation</h3>
|
||
|
||
<ul class="list-style-none d-flex flex-justify-center flex-wrap mb-2 mb-lg-0" aria-labelledby="sr-footer-heading">
|
||
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to Terms","label":"text:terms"}" href="https://docs.github.com/site-policy/github-terms/github-terms-of-service" data-view-component="true" class="Link--secondary Link">Terms</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to privacy","label":"text:privacy"}" href="https://docs.github.com/site-policy/privacy-policies/github-privacy-statement" data-view-component="true" class="Link--secondary Link">Privacy</a>
|
||
</li>
|
||
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to security","label":"text:security"}" href="https://github.com/security" data-view-component="true" class="Link--secondary Link">Security</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to status","label":"text:status"}" href="https://www.githubstatus.com/" data-view-component="true" class="Link--secondary Link">Status</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to community","label":"text:community"}" href="https://github.community/" data-view-component="true" class="Link--secondary Link">Community</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to docs","label":"text:docs"}" href="https://docs.github.com/" data-view-component="true" class="Link--secondary Link">Docs</a>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<a data-analytics-event="{"category":"Footer","action":"go to contact","label":"text:contact"}" href="https://support.github.com?tags=dotcom-footer" data-view-component="true" class="Link--secondary Link">Contact</a>
|
||
</li>
|
||
|
||
|
||
<li class="mx-2" >
|
||
<cookie-consent-link>
|
||
<button
|
||
type="button"
|
||
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent"
|
||
data-action="click:cookie-consent-link#showConsentManagement"
|
||
data-analytics-event="{"location":"footer","action":"cookies","context":"subfooter","tag":"link","label":"cookies_link_subfooter_footer"}"
|
||
>
|
||
Manage cookies
|
||
</button>
|
||
</cookie-consent-link>
|
||
</li>
|
||
|
||
<li class="mx-2">
|
||
<cookie-consent-link>
|
||
<button
|
||
type="button"
|
||
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent text-left"
|
||
data-action="click:cookie-consent-link#showConsentManagement"
|
||
data-analytics-event="{"location":"footer","action":"dont_share_info","context":"subfooter","tag":"link","label":"dont_share_info_link_subfooter_footer"}"
|
||
>
|
||
Do not share my personal information
|
||
</button>
|
||
</cookie-consent-link>
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
|
||
<ghcc-consent id="ghcc" class="position-fixed bottom-0 left-0" style="z-index: 999999"
|
||
data-locale="en"
|
||
data-initial-cookie-consent-allowed=""
|
||
data-cookie-consent-required="false"
|
||
></ghcc-consent>
|
||
|
||
|
||
|
||
|
||
<div id="ajax-error-message" class="ajax-error-message flash flash-error" hidden>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
|
||
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
|
||
</svg>
|
||
<button type="button" class="flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
You can’t perform that action at this time.
|
||
</div>
|
||
|
||
<template id="site-details-dialog">
|
||
<details class="details-reset details-overlay details-overlay-dark lh-default color-fg-default hx_rsm" open>
|
||
<summary role="button" aria-label="Close dialog"></summary>
|
||
<details-dialog class="Box Box--overlay d-flex flex-column anim-fade-in fast hx_rsm-dialog hx_rsm-modal">
|
||
<button class="Box-btn-octicon m-0 btn-octicon position-absolute right-0 top-0" type="button" aria-label="Close dialog" data-close-dialog>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
|
||
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
|
||
</svg>
|
||
</button>
|
||
<div class="octocat-spinner tmp-my-6 js-details-dialog-spinner"></div>
|
||
</details-dialog>
|
||
</details>
|
||
</template>
|
||
|
||
<div class="Popover js-hovercard-content position-absolute" style="display: none; outline: none;">
|
||
<div class="Popover-message Popover-message--bottom-left Popover-message--large Box color-shadow-large" style="width:360px;">
|
||
</div>
|
||
</div>
|
||
|
||
<template id="snippet-clipboard-copy-button">
|
||
<div class="zeroclipboard-container position-absolute right-0 top-0">
|
||
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn js-clipboard-copy m-2 p-0" data-copy-feedback="Copied!" data-tooltip-direction="w">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon m-2 tmp-m-2">
|
||
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
|
||
</svg>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none m-2 tmp-m-2">
|
||
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
|
||
</svg>
|
||
</clipboard-copy>
|
||
</div>
|
||
</template>
|
||
<template id="snippet-clipboard-copy-button-unpositioned">
|
||
<div class="zeroclipboard-container">
|
||
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w">
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon">
|
||
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
|
||
</svg>
|
||
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none">
|
||
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
|
||
</svg>
|
||
</clipboard-copy>
|
||
</div>
|
||
</template>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
<div id="js-global-screen-reader-notice" class="sr-only mt-n1" aria-live="polite" aria-atomic="true" ></div>
|
||
<div id="js-global-screen-reader-notice-assertive" class="sr-only mt-n1" aria-live="assertive" aria-atomic="true"></div>
|
||
</body>
|
||
</html>
|
||
|