Files
nexus/sreweekly/articles/93/02-distsys-class-readme-markdown-at-master-aphyr-distsys-class-github.html
2026-09-12 17:23:01 +08:00

3985 lines
826 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html
lang="en"
data-color-mode="auto" data-light-theme="light" data-dark-theme="dark"
data-a11y-animated-images="system" data-a11y-link-underlines="true"
>
<head>
<meta charset="utf-8">
<link rel="dns-prefetch" href="https://github.githubassets.com">
<link rel="dns-prefetch" href="https://avatars.githubusercontent.com">
<link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
<link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
<link rel="preconnect" href="https://github.githubassets.com" crossorigin>
<link rel="preconnect" href="https://avatars.githubusercontent.com">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="light" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light-99f877e9ddfc0e51.css" /><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-48fdd0811afbab3c.css" /><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-f4bf1142976e4bbf.css" /><link data-color-theme="light_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind_high_contrast-f661b49995ba0bd8.css" /><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-0b38d22346321c92.css" /><link data-color-theme="light_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia_high_contrast-f1c62c9e70259b9f.css" /><link data-color-theme="dark" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark-79ad2ace604703b3.css" /><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-24484a076f02295f.css" /><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-f50cacf0a86b9929.css" /><link data-color-theme="dark_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind_high_contrast-e61d4f4ca17852c2.css" /><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-39c10993d5603fac.css" /><link data-color-theme="dark_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia_high_contrast-73236c840c0c7d90.css" /><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-0de76f07cc035b10.css" /><link data-color-theme="dark_dimmed_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed_high_contrast-fd1500c8744e40d6.css" />
<style type="text/css">
:root {
--tab-size-preference: 4;
}
pre, code {
tab-size: var(--tab-size-preference);
}
</style>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-ed9ca172356fd545.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-1d2c7f7b52a6068b.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-adcabba7b5c5d221.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-552513ad07a183a1.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repository-11ee8a031c040c1a.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-2d56bdb0166c0238.css" />
<script type="application/json" id="client-env">{"locale":"en","featureFlags":["actions_enable_background_steps","activity_diff_file_tree","activity_repos_file_tree","activity_repos_overview_header","activity_repos_overview_sidebar","agent_author_search_expansion","agent_author_search_expansion_ui_pulls","alternate_user_config_repo","billing_billable_licenses_cost_center_bucket_fix","billing_budget_expiration","billing_copilot_license_budget_use_license_count","billing_cost_center_list_assigned_resources","billing_discount_threshold_notification","block_user_close_content","cca_show_aic_usage_per_session","ccr_files_changed_model_picker","code_quality_enablement_banner_targeting","code_quality_remove_preview","code_view_raf_sticky_lines","codespaces_prebuild_region_target_update","coding_agent_third_party_model_ui","contentful_primer_code_blocks","copilot_agent_snippy","copilot_api_agentic_issue_marshal_yaml","copilot_automations_pagination","copilot_chat_clear_model_selection_for_default_change","copilot_chat_compact_tables","copilot_chat_header_reorder","copilot_chat_max_upsell","copilot_chat_minimize_contextual","copilot_chat_model_picker_promotions","copilot_chat_models_browser_cache","copilot_chat_new_topic_nudge","copilot_chat_reduce_quota_checks","copilot_chat_vision_dotcom_chat_ga_gate","copilot_chat_vision_preview_gate","copilot_css_textarea_autosize","copilot_custom_copilots","copilot_custom_copilots_feature_preview","copilot_diff_reference_context","copilot_duplicate_thread","copilot_extensions_removal_on_marketplace","copilot_fix_failed_workflows_all_skus","copilot_ftp_hyperspace_upgrade_prompt","copilot_hide_hovercard","copilot_immersive_code_block_transition_wrap","copilot_immersive_embedded_draggable","copilot_immersive_embedded_header_button","copilot_immersive_file_block_transition_open","copilot_immersive_file_preview_keep_mounted","copilot_immersive_suggestion_pills","copilot_immersive_task_hyperlinking","copilot_immersive_task_within_chat_thread","copilot_mc_cli_resume_any_users_task","copilot_mission_control_agent_merge_fix_ci","copilot_mission_control_agent_merge_resolve_conflicts","copilot_mission_control_early_stop","copilot_mission_control_environment_list_icons","copilot_mission_control_managed_sandbox_environments","copilot_mission_control_needs_attention","copilot_mission_control_reasoning_effort","copilot_mission_control_sandbox_client_side_clone","copilot_mission_control_sandbox_remote_bypass","copilot_mission_control_session_filters","copilot_mission_control_task_alive_updates","copilot_mission_control_task_sharing","copilot_org_policy_page_focus_mode","copilot_pr_chat_enhancements","copilot_prominent_upgrade_button","copilot_resource_panel","copilot_share_active_subthread","copilot_spaces_ga","copilot_spaces_individual_policies_ga","copilot_spark_handle_nil_friendly_name","copilot_swe_agent_authorization_status_ui","copilot_swe_agent_automation_resource_scoped_writes","copilot_swe_agent_hide_model_picker_if_only_auto","copilot_swe_agent_issue_comment_trigger","copilot_swe_agent_pr_comment_model_picker","copilot_swe_agent_pull_request_comment_trigger","copilot_swe_agent_pull_request_merged_trigger","copilot_swe_agent_pull_request_opened_trigger","copilot_swe_agent_pull_request_synchronize_trigger","copilot_swe_agent_use_subagents","copilot_task_api_github_rest_style","copilot_task_scoped_alive_channel","copilot_token_based_billing","copilot_unconfigured_is_inherited","copilot_user_can_upgrade_plan_field","copilot_workbench_sunset","copilot_workbench_sunset_redirect","copilot_workbench_ubb","dashboard_indexeddb_caching","dashboard_lists_max_age_filter","dashboard_universe_2025_feedback_dialog","flex_cta_groups_mvp","flex_suite_disable_river_accordion_dither","glc_code_quality_repo_settings_workflow_config","hide_github_models_ui","hyperspace_2025_logged_out_batch_1","hyperspace_2025_logged_out_batch_2","hyperspace_2025_logged_out_batch_3","in_product_messaging_datadog_monitoring","ipm_global_transactional_message_copilot","ipm_global_transactional_message_issues","ipm_global_transactional_message_prs","ipm_global_transactional_message_repos","ipm_global_transactional_message_spaces","issue_fields_multi_select","issue_inline_avatars","issue_pinned_views","issue_pinned_views_optimistic_updates","issue_relative_time_micro","issues_dashboard_sso_structured_errors","issues_expanded_file_types","issues_hide_closed_sub_issues","issues_lazy_load_comment_box_suggestions","issues_react_chrome_container_query_fix","labels_archiving","labels_archiving_info","landing_pages_ninetailed","landing_pages_web_vitals_tracking","lifecycle_label_name_updates","marketing_pages_search_explore_provider","memex_default_issue_create_repository","memex_lazy_hydrate_agent_tasks","memex_live_update_hovercard","memex_mwl_filter_field_delimiter","memex_remove_deprecated_type_issue","merge_queue_restricted_pushers_warning","merge_status_checks_refetch_dedupe","merge_status_header_feedback","oauth_authorize_clickjacking_protection","octocaptcha_origin_optimization","primer_react_css_anchor_positioning","primer_react_merged_forwarded_refs","property_definition_empty_state_suggestions","prs_copilot_app_open_action","prs_css_anchor_positioning","prs_new_conversation_comments_api","prs_omit_files_routes","pull_request_copilot_attribution_header","pull_request_overview_panel_edit_description","pull_request_persister","pull_request_stacks_feedback_dialog","pull_request_stacks_rebase_conflict_instructions","pull_request_virtualization_image_estimate","pull_request_virtualization_scroll_compensation","pull_request_virtualization_scroll_intent","pulls_dashboard_sidebar_layout","quick_search_lazy_suggestions","react_blob_isolate_code_lines","react_blob_ssr_content_visibility","react_data_router_tanstack_allowed","react_query_props_with_key","react_sandbox_future_tanstack","repo_app_actions_workflows","repo_app_agents","repo_app_commits","repo_app_issues","repo_app_pull_requests","repo_app_turbo","repo_issues_sidebar_layout","repo_overview_ask_copilot","repos_contributors_limited_default_range","review_involves_filter","rule_ignored_file_paths","rulesets_actor_list_editor","sample_network_conn_type","security_center_artifact_filters_popover","see_who_reacted","semantic_similarity_duplicate_issue_detection","session_logs_ungroup_reasoning_text","set_sha256_on_repo_creation_form","site_banner_desktop_copilot_app","site_ghca_pixel_mona","site_github_app_business_enterprise_signup_ctas","site_github_app_ga_page","site_github_app_ga_page_highlight","site_github_app_mobile_native_share","site_global_banner_copilot_day","site_global_banner_dev_days_attendee","site_global_banner_learn_copilot_sdk","site_global_nav_spark_models_removed","spark_prompt_secret_scanning","spark_server_connection_status","suppress_automated_browser_vitals","swp_forms_disable_octocaptcha","thread_resolution_reason","ui_service_compute_proxy_control","ui_service_referrer_metrics","update_issue_suggestions","viewscreen_sandbox","warn_inaccessible_attachments","webp_support","workbench_store_readonly"],"githubDomain":"https://github.com","copilotApiOverrideUrl":"https://api.githubcopilot.com","cmcApiUrl":"https://api.github.com/cmc_internal/api"}</script>
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/high-contrast-cookie-e3d808ee18eb9784.js"></script>
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wp-runtime-f176a19df481ff1f.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-foundation-13e668e8647c754c.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/app-runtime-79f2476139a75354.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fetch-utilities-5305162aa9984914.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ser-cd1b421ad0ad7ed5.js" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/environment-04f6e0170488e4e8.js" defer="defer"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/app-runtime.7a9535dabdf3b702.module.css" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/catalyst-044fa3a2acdaf0a9.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/selector-observer-e88088f989b27670.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/relative-time-element-85cb37305f54dec4.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/by-6b3c07383371bf58.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ja9-36f036c13c2e3a4c.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/kw-866fd9513ae95106.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/r1-b075aab3204f32f9.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/jz5-7c5d1669717041f8.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hj-5126390ff433704d.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j05-dce4d588268f0f7e.js" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/github-elements-bc32978e9af6a818.js" defer="defer"></script>
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/element-registry-c9988048f4d520db.js" defer="defer"></script>
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/runtime-helpers-f3f5d71440009c48.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aria-live-76b18916d0c8ec4d.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/hotkey-5109c77d7ac61078.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-core-7fa41ffc30596003.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/react-lib-33858e668cc159ef.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/04-2e0b5b7403cb7b99.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ur-024971cb63acad59.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4t-9933bcccf9a9524f.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/6n-71c0ae3e489775ef.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/yhq-3a209035a30b1a09.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/z1-43f07743b557bfbe.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wo-09f85d22a6c8931b.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/9y-75e8f8c6f5eccd49.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j0-e9bfed51669329e8.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0p-c8fed99e921442ca.js" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/behaviors-28be759561a76633.js" defer="defer"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/react-core.a447335ec9ae046d.module.css" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-menu-bc6f54087811ea2f.js" defer="defer"></script>
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/primer-react-fad6c97f07e0280a.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/aq-0dbf7d12b4d37e14.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j6-1e8b001d24ed9cac.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/t7l-0a8b07066f584f39.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fe-7576d9b281e2106a.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/wsr-5fd3071e5a9061ab.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ps6-85a59b16b0ceea7e.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xv-51cb34754bda73a9.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/n4u-6317ce1c31aed1db.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/j8s-d05eedc69cebc200.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/la-de176c4ee471f2b3.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/u1-c2354415da044847.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/os-1dee58ca1675cc41.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/50-a6c15ac2de24205d.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0o-6dedd6c7dfa5a7b9.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nsi-89f8558c22a1e954.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3np-b8ac617a3f77670d.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/on-75d8015f7c16be87.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bi0-fa59350332fdd7ab.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/b1-e65db5e6cf097025.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0l5-16b0cf9b35fe7f75.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/c2-fb6736e73d800b4e.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/f9-6f78c2d90591da73.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0k-f05429fb3eb4e9a3.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/sp-2bfa60fc59b501fb.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/og-78525ede7a2add85.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/q5-23ee69a34fd8c463.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/qod-dbecf5f56ddaa69a.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/du-f925771a20e38027.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/ql0-324fc36dae3b9a42.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/20x-dad6ee9ff6177544.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/dsj-342cb45d382b6fdb.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/6f-ddbc6e6b37135f96.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/e84-85a4f8740b573283.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0pz-7d0eea7df68b2d64.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7mh-b8dbc13ace044f4c.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/bsb-4f4ac8c20da73fac.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/46-2db674c6bc8272ab.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/q1-590dbf4434fdc3ad.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/me-6d77019ac8eb6290.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/0d-623095bb226c1f11.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/xc-7fd1e6996fa23469.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/7c-42dc52617c4fd396.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/4b-cfa20397aa7c0823.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/dynamic-github-ui--code-view--route-components-50d91ad640564ff8.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/gt-b75626ea0831d606.js" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/code-view-b106d3781f6c832b.js" defer="defer"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/ql0.16c145071702f131.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/46.73d41df1aa9a8bca.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/xc.678a3a89a0b97d69.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/4b.b27df37732ef81c7.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dynamic-github-ui--code-view--route-components.c7b328225801cde0.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-view.292547eba0d657d3.module.css" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/18-30bd8f55db757032.js" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/31k-1c94735cfe3b279c.js" />
<script crossorigin="anonymous" type="module" src="https://github.githubassets.com/assets/notifications-subscriptions-menu-6ad3cbf09782cc71.js" defer="defer"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.ac49a7696ff96455.module.css" />
<title>distsys-class/README.markdown at master · aphyr/distsys-class · GitHub</title>
<meta name="route-pattern" content="/:user_id/:repository/blob/*name(/*path)" data-turbo-transient>
<meta name="route-controller" content="blob" data-turbo-transient>
<meta name="route-action" content="show" data-turbo-transient>
<meta name="fetch-nonce" content="v2:ae2859f0-21d8-334f-de11-9d4b89d2a997">
<meta name="current-catalog-service-hash" content="f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb">
<meta name="request-id" content="C2CC:1662:DDB346:EA1049:6AA3719F" data-pjax-transient="true"/><meta name="html-safe-nonce" content="55b110e0def4ee88ae1a54deb5ef1b8e9cae1cd728af85fb33a2e9377a11ab9d" data-pjax-transient="true"/><meta name="visitor-payload" content="eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJDMkNDOjE2NjI6RERCMzQ2OkVBMTA0OTo2QUEzNzE5RiIsInZpc2l0b3JfaWQiOiI1MjcxMTQ3OTEzMTYzMjcyNjA3IiwicmVnaW9uX2VkZ2UiOiJzZWEiLCJyZWdpb25fcmVuZGVyIjoic2VhIn0=" data-pjax-transient="true"/><meta name="visitor-hmac" content="8fcd4afd4eae0f7709f0b99177076bcc30556c908624b8a6f4d9e22fb27ca11a" data-pjax-transient="true"/>
<meta name="hovercard-subject-tag" content="repository:55260959" data-turbo-transient>
<meta name="github-keyboard-shortcuts" content="repository,source-code,file-tree,copilot" data-turbo-transient="true" />
<meta name="selected-link" value="repo_source" data-turbo-transient>
<link rel="assets" href="https://github.githubassets.com/">
<meta name="google-site-verification" content="Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I">
<meta name="octolytics-url" content="https://collector.github.com/github/collect" />
<meta name="analytics-location" content="/&lt;user-name&gt;/&lt;repo-name&gt;/blob/show" data-turbo-transient="true" />
<meta name="user-login" content="">
<meta name="viewport" content="width=device-width">
<meta name="description" content="Class materials for a distributed systems lecture series - distsys-class/README.markdown at master · aphyr/distsys-class">
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
<link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
<meta property="fb:app_id" content="1401488693436528">
<meta name="apple-itunes-app" content="app-id=1477376905, app-argument=https://github.com/aphyr/distsys-class/blob/master/README.markdown" />
<meta name="twitter:image" content="https://opengraph.githubassets.com/61289414d6759d76e1eda5285df8b8c23e8e447b3ce0542aaeaddec7977ab579/aphyr/distsys-class" /><meta name="twitter:site" content="@github" /><meta name="twitter:card" content="summary_large_image" /><meta name="twitter:title" content="distsys-class/README.markdown at master · aphyr/distsys-class" /><meta name="twitter:description" content="Class materials for a distributed systems lecture series - aphyr/distsys-class" />
<meta property="og:image" content="https://opengraph.githubassets.com/61289414d6759d76e1eda5285df8b8c23e8e447b3ce0542aaeaddec7977ab579/aphyr/distsys-class" /><meta property="og:image:alt" content="Class materials for a distributed systems lecture series - aphyr/distsys-class" /><meta property="og:image:width" content="1200" /><meta property="og:image:height" content="600" /><meta property="og:site_name" content="GitHub" /><meta property="og:type" content="object" /><meta property="og:title" content="distsys-class/README.markdown at master · aphyr/distsys-class" /><meta property="og:url" content="https://github.com/aphyr/distsys-class/blob/master/README.markdown" /><meta property="og:description" content="Class materials for a distributed systems lecture series - aphyr/distsys-class" />
<meta name="hostname" content="github.com">
<meta name="expected-hostname" content="github.com">
<meta http-equiv="x-pjax-version" content="d1d04ca6665942cf286c040b445fa3bb2efa681595d1b20c896a813eab8249c0" data-turbo-track="reload">
<meta http-equiv="x-pjax-csp-version" content="2af6a6627810d190be616096b617c8f37a1419b9435f76190f0e0d7638222ac1" data-turbo-track="reload">
<meta http-equiv="x-pjax-css-version" content="b7a4653359baeee3d827ca91d7c149e4962234557341efae58fad4aa5ce82a0a" data-turbo-track="reload">
<meta http-equiv="x-pjax-js-version" content="2d107c02f094199e148d05393ad004e0659898f4132ff3fd74a5c395c6857435" data-turbo-track="reload">
<meta name="turbo-cache-control" content="no-preview" data-turbo-transient="">
<meta name="turbo-cache-control" content="no-cache" data-turbo-transient>
<meta data-hydrostats="publish">
<meta name="go-import" content="github.com/aphyr/distsys-class git https://github.com/aphyr/distsys-class.git">
<meta name="octolytics-dimension-user_id" content="3748" /><meta name="octolytics-dimension-user_login" content="aphyr" /><meta name="octolytics-dimension-repository_id" content="55260959" /><meta name="octolytics-dimension-repository_nwo" content="aphyr/distsys-class" /><meta name="octolytics-dimension-repository_public" content="true" /><meta name="octolytics-dimension-repository_is_fork" content="false" /><meta name="octolytics-dimension-repository_network_root_id" content="55260959" /><meta name="octolytics-dimension-repository_network_root_nwo" content="aphyr/distsys-class" />
<meta name="turbo-body-classes" content="logged-out env-production page-responsive">
<meta name="disable-turbo" content="false">
<meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
<meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
<meta name="release" content="a0ac789986b5b2f33d10a9645118fd3651917b88" data-turbo-track="reload">
<meta name="ui-target" content="full">
<link rel="mask-icon" href="https://github.githubassets.com/assets/pinned-octocat-093da3e6fa40.svg" color="#000000">
<link rel="alternate icon" class="js-site-favicon" type="image/png" href="https://github.githubassets.com/favicons/favicon.png">
<link rel="icon" class="js-site-favicon" type="image/svg+xml" href="https://github.githubassets.com/favicons/favicon.svg" data-base-href="https://github.githubassets.com/favicons/favicon">
<meta name="theme-color" content="#1e2327">
<meta name="color-scheme" content="light dark" />
<link rel="manifest" href="/manifest.json" crossOrigin="use-credentials">
</head>
<body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
<div data-turbo-body class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
<div id="__primerPortalRoot__" style="z-index: 1000; position: absolute; width: 100%;" data-turbo-permanent></div>
<div class="position-relative header-wrapper js-header-wrapper ">
<a href="#start-of-content" data-skip-target-assigned="false" class="px-2 tmp-py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content">Skip to content</a>
<span data-view-component="true" class="progress-pjax-loader Progress position-fixed width-full">
<span style="width: 0%;" data-view-component="true" class="Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis"></span>
</span>
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/obo5-8a5f0070d148d90b.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-c36ec12975d77de7.js" fetchpriority="low" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog.8302d893d5bf61ba.module.css" />
<react-partial
partial-name="keyboard-shortcuts-dialog"
data-ssr="false"
data-attempted-ssr="false"
data-react-profiling="false"
>
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"docsUrl":"https://docs.github.com/get-started/accessibility/keyboard-shortcuts"}}</script>
<div data-target="react-partial.reactRoot"></div>
</react-partial>
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/nvz-d8ae5b1b70e399a7.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/mz-504eb598d2e57128.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/fz-30a6af7c53dfa4f1.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/lht-d07b466cb387fec6.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/1lf-a0b5bbf5742c0fd3.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/3d-82c347d20a13fb90.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/tua-07c0f2fd28675950.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/lazy-react-partial-marketing-header-866751d51954477d.js" fetchpriority="low" />
<link crossorigin="anonymous" rel="modulepreload" href="https://github.githubassets.com/assets/marketing-header-03f048a3b4907614.js" fetchpriority="low" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-css.3ff9ad885fbbd7b1.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react-brand-css.206bbd4fd232654e.module.css" />
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/lazy-react-partial-marketing-header.784f79e59a54ffa6.module.css" />
<react-partial
partial-name="marketing-header"
data-ssr="true"
data-attempted-ssr="true"
data-react-profiling="false"
>
<script type="application/json" data-target="react-partial.embeddedData">{"props":{"color_mode":"dark","logged_in":false,"marketing_page":false,"home_path":"/","login_path":"/login?return_to=https%3A%2F%2Fgithub.com%2Faphyr%2Fdistsys-class%2Fblob%2Fmaster%2FREADME.markdown","signup_path":"/signup?ref_cta=Sign+up\u0026ref_loc=header+logged+out\u0026ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow\u0026source=header-repo\u0026source_repo=aphyr%2Fdistsys-class","signup_enabled":true,"is_signup_controller":false,"show_search_and_nav":true,"hide_search":false,"private_mode_enabled":false,"should_use_dotcom_links":true,"auth_hydro_click":"{\"event_type\":\"authentication.click\",\"payload\":{\"location_in_page\":\"site header menu\",\"repository_id\":null,\"auth_type\":\"SIGN_UP\",\"originating_url\":\"https://github.com/aphyr/distsys-class/blob/master/README.markdown\",\"user_id\":null}}","auth_hydro_click_hmac":"b8e6ccc4f18d35e5477a4d151c3f567aa0a54465d1c785cf459c1ec94acfa70e","overlay":false,"fixed":false}}</script>
<div data-target="react-partial.reactRoot"><div data-color-mode="dark" data-light-theme="light" data-dark-theme="dark"><header class="MarketingHeader-module__root__Tk7n3 HeaderMktg header-logged-out" role="banner" data-marketing-header="true" data-color-mode="dark" data-light-theme="light" data-dark-theme="dark" data-is-top="true"><h2 class="MarketingHeader-module__visuallyHidden__sqKsl">Navigation Menu</h2><button type="button" class="MarketingHeader-module__backdrop__sw4RU" aria-label="Close navigation menu"></button><div class="MarketingHeader-module__bar__mBSyE"><div class="MarketingHeader-module__topRow__yeury"><div class="MarketingHeader-module__toggleSlot__hDxbh"><button type="button" class="HeaderMenuToggle-module__toggle__i8EiC" aria-label="Toggle navigation" aria-expanded="false"><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span><span class="HeaderMenuToggle-module__toggleBar__jVN0H"></span></button></div><a href="/" aria-label="Homepage" class="HeaderLogo-module__logo__UFyHI" data-analytics-event="{&quot;action&quot;:&quot;homepage&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;logo&quot;,&quot;location&quot;:&quot;header&quot;,&quot;label&quot;:&quot;homepage_link_logo_header&quot;}"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github" viewBox="0 0 24 24" width="32" height="32" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path></svg></a><div class="AuthCTAs-module__mobileActions__NNzeV"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/login?return_to=https%3A%2F%2Fgithub.com%2Faphyr%2Fdistsys-class%2Fblob%2Fmaster%2FREADME.markdown" data-analytics-event="{&quot;action&quot;:&quot;sign_in&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;auth_cta&quot;,&quot;location&quot;:&quot;header&quot;,&quot;label&quot;:&quot;sign_in_link_auth_cta_header&quot;}" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="b8e6ccc4f18d35e5477a4d151c3f567aa0a54465d1c785cf459c1ec94acfa70e"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_3dd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_3dd_">Appearance settings</div></div></div><div class="MarketingHeader-module__menu__GIy3y"><div class="MarketingHeader-module__menuWrapper__owstH"><nav class="MarketingNavigation-module__nav__W0KYY" aria-label="Global"><ul class="MarketingNavigation-module__list__tFbMb"><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_nd_">Platform<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5knd_">AI CODE CREATION</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot" data-analytics-event="{&quot;action&quot;:&quot;github_copilot&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_copilot_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>GitHub Copilot</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Write better code with AI</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/ai/github-app" data-analytics-event="{&quot;action&quot;:&quot;github_copilot_app&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_copilot_app_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mark-github NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.766 11.328c-2.063-.25-3.516-1.734-3.516-3.656 0-.781.281-1.625.75-2.188-.203-.515-.172-1.609.063-2.062.625-.078 1.468.25 1.968.703.594-.187 1.219-.281 1.985-.281.765 0 1.39.094 1.953.265.484-.437 1.344-.765 1.969-.687.218.422.25 1.515.046 2.047.5.593.766 1.39.766 2.203 0 1.922-1.453 3.375-3.547 3.64.531.344.89 1.094.89 1.954v1.625c0 .468.391.734.86.547C13.781 14.359 16 11.53 16 8.03 16 3.61 12.406 0 7.984 0 3.563 0 0 3.61 0 8.031a7.88 7.88 0 0 0 5.172 7.422c.422.156.828-.125.828-.547v-1.25c-.219.094-.5.156-.75.156-1.031 0-1.64-.562-2.078-1.609-.172-.422-.36-.672-.719-.719-.187-.015-.25-.093-.25-.187 0-.188.313-.328.625-.328.453 0 .844.281 1.25.86.313.452.64.655 1.031.655s.641-.14 1-.5c.266-.265.47-.5.657-.656"></path></svg>GitHub Copilot app</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Direct agents from issue to merge</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/mcp" data-analytics-event="{&quot;action&quot;:&quot;mcp_registry&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;mcp_registry_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-mcp NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.52 1.12a3.578 3.578 0 0 1 6.078 2.98 3.578 3.578 0 0 1 2.982 6.08l-3.292 3.293a.252.252 0 0 0 0 .354l.843.843a.749.749 0 1 1-1.06 1.06l-.844-.843a1.75 1.75 0 0 1 0-2.474L13.52 9.12a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0L7.731 9.03A.75.75 0 0 1 6.67 7.97l2.85-2.85a2.08 2.08 0 0 0 0-2.94 2.08 2.08 0 0 0-2.94 0l-4.799 4.8A.75.75 0 0 1 .72 5.92Z"></path><path d="M7.52 3.12a.749.749 0 1 1 1.06 1.06L5.731 7.03A2.079 2.079 0 0 0 8.67 9.97l2.85-2.85a.749.749 0 1 1 1.06 1.06l-2.849 2.85A3.578 3.578 0 0 1 4.67 5.97Z"></path></svg>MCP Registry</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Integrate external tools</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9knd_">DEVELOPER WORKFLOWS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9knd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/actions" data-analytics-event="{&quot;action&quot;:&quot;actions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;actions_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-workflow NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path></svg>Actions</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Automate any workflow</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/codespaces" data-analytics-event="{&quot;action&quot;:&quot;codespaces&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;codespaces_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codespaces NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 11.25c0-.966.784-1.75 1.75-1.75h12.5c.966 0 1.75.784 1.75 1.75v3A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm2-9.5C2 .784 2.784 0 3.75 0h8.5C13.216 0 14 .784 14 1.75v5a1.75 1.75 0 0 1-1.75 1.75h-8.5A1.75 1.75 0 0 1 2 6.75Zm1.75-.25a.25.25 0 0 0-.25.25v5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5a.25.25 0 0 0-.25-.25Zm-2 9.5a.25.25 0 0 0-.25.25v3c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-3a.25.25 0 0 0-.25-.25Z"></path><path d="M7 12.75a.75.75 0 0 1 .75-.75h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path></svg>Codespaces</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Instant dev environments</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/issues" data-analytics-event="{&quot;action&quot;:&quot;issues&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;issues_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-issue-opened NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path></svg>Issues</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Plan and track work</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-review" data-analytics-event="{&quot;action&quot;:&quot;code_review&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;code_review_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path></svg>Code Review</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Manage code changes</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/code-quality" data-analytics-event="{&quot;action&quot;:&quot;code_quality&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;code_quality_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-codescan-checkmark NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.28 6.28a.75.75 0 1 0-1.06-1.06L6.25 8.19l-.97-.97a.75.75 0 0 0-1.06 1.06l1.5 1.5a.75.75 0 0 0 1.06 0l3.5-3.5Z"></path><path d="M7.5 15a7.5 7.5 0 1 1 5.807-2.754l2.473 2.474a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2.474-2.473A7.472 7.472 0 0 1 7.5 15Zm0-13.5a6 6 0 1 0 4.094 10.386.748.748 0 0 1 .293-.292 6.002 6.002 0 0 0 1.117-6.486A6.002 6.002 0 0 0 7.5 1.5Z"></path></svg>Code Quality</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enforce quality at merge</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dknd_">APPLICATION SECURITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{&quot;action&quot;:&quot;github_advanced_security&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_advanced_security_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Find and fix vulnerabilities</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/code-security" data-analytics-event="{&quot;action&quot;:&quot;code_security&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;code_security_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-code-square NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>Code security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Secure your code as you build</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security/secret-protection" data-analytics-event="{&quot;action&quot;:&quot;secret_protection&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;secret_protection_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-lock NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M4 4a4 4 0 0 1 8 0v2h.25c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 12.25 15h-8.5A1.75 1.75 0 0 1 2 13.25v-5.5C2 6.784 2.784 6 3.75 6H4Zm8.25 3.5h-8.5a.25.25 0 0 0-.25.25v5.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25ZM10.5 6V4a2.5 2.5 0 1 0-5 0v2Z"></path></svg>Secret protection</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Stop leaks before they start</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ NavGroup-module__hasSeparator__FnMrN"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_hknd_">EXPLORE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_hknd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/why-github" data-analytics-event="{&quot;action&quot;:&quot;why_github&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;why_github_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Why GitHub</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{&quot;action&quot;:&quot;documentation&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;documentation_link_platform_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog" data-analytics-event="{&quot;action&quot;:&quot;blog&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;blog_link_platform_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Blog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://github.blog/changelog" data-analytics-event="{&quot;action&quot;:&quot;changelog&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;changelog_link_platform_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Changelog</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/marketplace" data-analytics-event="{&quot;action&quot;:&quot;marketplace&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;marketplace_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Marketplace</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/features" data-analytics-event="{&quot;action&quot;:&quot;view_all_features&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;platform&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_features_link_platform_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all features</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_17d_">Solutions<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_17d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5l7d_">BY COMPANY SIZE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise" data-analytics-event="{&quot;action&quot;:&quot;enterprises&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;enterprises_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Enterprises</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/team" data-analytics-event="{&quot;action&quot;:&quot;small_and_medium_teams&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;small_and_medium_teams_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Small and medium teams</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/enterprise/startups" data-analytics-event="{&quot;action&quot;:&quot;startups&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;startups_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Startups</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/nonprofits" data-analytics-event="{&quot;action&quot;:&quot;nonprofits&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;nonprofits_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Nonprofits</span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9l7d_">BY USE CASE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9l7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/app-modernization" data-analytics-event="{&quot;action&quot;:&quot;app_modernization&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;app_modernization_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">App Modernization</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devsecops" data-analytics-event="{&quot;action&quot;:&quot;devsecops&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;devsecops_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevSecOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/devops" data-analytics-event="{&quot;action&quot;:&quot;devops&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;devops_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/use-case/ci-cd" data-analytics-event="{&quot;action&quot;:&quot;ci/cd&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;ci/cd_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">CI/CD</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/use-case" data-analytics-event="{&quot;action&quot;:&quot;view_all_use_cases&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_use_cases_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all use cases</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dl7d_">BY INDUSTRY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dl7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/healthcare" data-analytics-event="{&quot;action&quot;:&quot;healthcare&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;healthcare_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Healthcare</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/financial-services" data-analytics-event="{&quot;action&quot;:&quot;financial_services&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;financial_services_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Financial services</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/manufacturing" data-analytics-event="{&quot;action&quot;:&quot;manufacturing&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;manufacturing_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Manufacturing</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/industry/government" data-analytics-event="{&quot;action&quot;:&quot;government&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;government_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Government</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions/industry" data-analytics-event="{&quot;action&quot;:&quot;view_all_industries&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_industries_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all industries</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/solutions" data-analytics-event="{&quot;action&quot;:&quot;view_all_solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_solutions_link_solutions_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all solutions</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_1nd_">Resources<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_1nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5lnd_">EXPLORE BY TOPIC</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=ai" data-analytics-event="{&quot;action&quot;:&quot;ai&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;ai_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">AI</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=software-development" data-analytics-event="{&quot;action&quot;:&quot;software_development&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;software_development_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Software Development</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=devops" data-analytics-event="{&quot;action&quot;:&quot;devops&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;devops_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">DevOps</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/articles?topic=security" data-analytics-event="{&quot;action&quot;:&quot;security&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;security_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources/articles" data-analytics-event="{&quot;action&quot;:&quot;view_all_topics&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_topics_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all topics</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9lnd_">EXPLORE BY TYPE</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9lnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/customer-stories" data-analytics-event="{&quot;action&quot;:&quot;customer_stories&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;customer_stories_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer stories</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/events" data-analytics-event="{&quot;action&quot;:&quot;events__webinars&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;events__webinars_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Events &amp; webinars</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/resources/whitepapers" data-analytics-event="{&quot;action&quot;:&quot;ebooks__reports&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;ebooks__reports_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Ebooks &amp; reports</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/solutions/executive-insights" data-analytics-event="{&quot;action&quot;:&quot;business_insights&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;business_insights_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Business insights</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://skills.github.com" data-analytics-event="{&quot;action&quot;:&quot;github_skills&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_skills_link_resources_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Skills</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dlnd_">SUPPORT &amp; SERVICES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dlnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://docs.github.com" data-analytics-event="{&quot;action&quot;:&quot;documentation&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;documentation_link_resources_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Documentation</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://support.github.com" data-analytics-event="{&quot;action&quot;:&quot;customer_support&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;customer_support_link_resources_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Customer support</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/orgs/community/discussions" data-analytics-event="{&quot;action&quot;:&quot;community_forum&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;community_forum_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Community forum</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trust-center" data-analytics-event="{&quot;action&quot;:&quot;trust_center&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;trust_center_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trust center</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/partners" data-analytics-event="{&quot;action&quot;:&quot;partners&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;partners_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Partners</span></a></li></ul></div></li></ul><div class="NavDropdown-module__trailingLinkContainer__VgJGL"><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--arrow-end___esdN8" href="https://github.com/resources" data-analytics-event="{&quot;action&quot;:&quot;view_all_resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;view_all_resources_link_resources_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">View all resources</span><svg class="Primer_Brand__ExpandableArrow-module__ExpandableArrow___aaZs9 Primer_Brand__Link-module__Link-arrow___yd78i" width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true" focusable="false"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="Primer_Brand__ExpandableArrow-module__ExpandableArrow-stem___0K8Hz" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg></a></div></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_27d_">Open Source<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_27d_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5m7d_">COMMUNITY</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/open-source/sponsors" data-analytics-event="{&quot;action&quot;:&quot;github_sponsors&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_sponsors_link_open_source_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sponsor-tiers NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.586 1C12.268 1 13.5 2.37 13.5 4.25c0 1.745-.996 3.359-2.622 4.831-.166.15-.336.297-.509.438l1.116 5.584a.75.75 0 0 1-.991.852l-2.409-.876a.25.25 0 0 0-.17 0l-2.409.876a.75.75 0 0 1-.991-.852L5.63 9.519a13.78 13.78 0 0 1-.51-.438C3.497 7.609 2.5 5.995 2.5 4.25 2.5 2.37 3.732 1 5.414 1c.963 0 1.843.403 2.474 1.073L8 2.198l.112-.125a3.385 3.385 0 0 1 2.283-1.068L10.586 1Zm-3.621 9.495-.718 3.594 1.155-.42a1.75 1.75 0 0 1 1.028-.051l.168.051 1.154.42-.718-3.592c-.199.13-.37.235-.505.314l-.169.097a.75.75 0 0 1-.72 0 9.54 9.54 0 0 1-.515-.308l-.16-.105ZM10.586 2.5c-.863 0-1.611.58-1.866 1.459-.209.721-1.231.721-1.44 0C7.025 3.08 6.277 2.5 5.414 2.5 4.598 2.5 4 3.165 4 4.25c0 1.23.786 2.504 2.128 3.719.49.443 1.018.846 1.546 1.198l.325.21.076-.047.251-.163a13.341 13.341 0 0 0 1.546-1.198C11.214 6.754 12 5.479 12 4.25c0-1.085-.598-1.75-1.414-1.75Z"></path></svg>GitHub Sponsors</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Fund open source developers</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9m7d_">PROGRAMS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9m7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://securitylab.github.com" data-analytics-event="{&quot;action&quot;:&quot;security_lab&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;security_lab_link_open_source_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Security Lab</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://maintainers.github.com" data-analytics-event="{&quot;action&quot;:&quot;maintainer_community&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;maintainer_community_link_open_source_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Maintainer Community</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://stars.github.com" data-analytics-event="{&quot;action&quot;:&quot;github_stars&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_stars_link_open_source_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">GitHub Stars</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 Primer_Brand__Link-module__Link--is-external___xsncV" href="https://archiveprogram.github.com" data-analytics-event="{&quot;action&quot;:&quot;archive_program&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;archive_program_link_open_source_navbar&quot;}" target="_blank" rel="noreferrer"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Archive Program</span><svg focusable="false" aria-label="External link" class="octicon octicon-link-external" role="img" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_dm7d_">REPOSITORIES</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_dm7d_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/topics" data-analytics-event="{&quot;action&quot;:&quot;topics&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;topics_link_open_source_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Topics</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/trending" data-analytics-event="{&quot;action&quot;:&quot;trending&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;trending_link_open_source_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Trending</span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0" href="https://github.com/collections" data-analytics-event="{&quot;action&quot;:&quot;collections&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;collections_link_open_source_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty">Collections</span></a></li></ul></div></li></ul></div></div></li><li><div class="NavDropdown-module__container__l2YeI"><button type="button" class="NavDropdown-module__button__PEHWX" aria-expanded="false" aria-controls="_R_2nd_">Enterprise<svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-right NavDropdown-module__buttonIcon__Tkl8_" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m6.427 4.427 3.396 3.396a.25.25 0 0 1 0 .354l-3.396 3.396A.25.25 0 0 1 6 11.396V4.604a.25.25 0 0 1 .427-.177Z"></path></svg></button><div id="_R_2nd_" class="NavDropdown-module__dropdown__xm1jd"><ul class="NavDropdown-module__list__zuCgG"><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_5mnd_">ENTERPRISE SOLUTIONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_5mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise" data-analytics-event="{&quot;action&quot;:&quot;enterprise_platform&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;enterprise_platform_link_enterprise_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-stack NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.122.392a1.75 1.75 0 0 1 1.756 0l5.003 2.902c.83.481.83 1.68 0 2.162L8.878 8.358a1.75 1.75 0 0 1-1.756 0L2.119 5.456a1.251 1.251 0 0 1 0-2.162ZM8.125 1.69a.248.248 0 0 0-.25 0l-4.63 2.685 4.63 2.685a.248.248 0 0 0 .25 0l4.63-2.685ZM1.601 7.789a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0L1.874 8.814A.75.75 0 0 1 1.6 7.789Zm0 3.5a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0l-5.248-3.044a.75.75 0 0 1-.273-1.025Z"></path></svg>Enterprise platform</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">AI-powered developer platform</span></span></a></li></ul></div></li><li><div class="NavGroup-module__group__W8SqJ"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--monospace___QXHDQ Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavGroup-module__title__Wzxz2" id="_R_9mnd_">AVAILABLE ADD-ONS</span><ul class="NavGroup-module__list__UCOFy" aria-labelledby="_R_9mnd_"><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/security/advanced-security" data-analytics-event="{&quot;action&quot;:&quot;github_advanced_security&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;github_advanced_security_link_enterprise_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-shield-check NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg>GitHub Advanced Security</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade security features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/features/copilot/copilot-business" data-analytics-event="{&quot;action&quot;:&quot;copilot_for_business&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;copilot_for_business_link_enterprise_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copilot NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path></svg>Copilot for Business</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade AI features</span></span></a></li><li><a class="Primer_Brand__Link-module__Link___lF11y Primer_Brand__Link-module__Link--default___VRVW0 NavLink-module__link__EG3d4" href="https://github.com/enterprise/premium-support" data-analytics-event="{&quot;action&quot;:&quot;premium_support&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;premium_support_link_enterprise_navbar&quot;}"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Link-module__Link--label___jM8Ty"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS Primer_Brand__Text-module__Text--weight-medium___qJKf_ NavLink-module__title__Q7t0p"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-comment-discussion NavLink-module__icon__ltGNM" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1h8.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 10.25 10H7.061l-2.574 2.573A1.458 1.458 0 0 1 2 11.543V10h-.25A1.75 1.75 0 0 1 0 8.25v-5.5C0 1.784.784 1 1.75 1ZM1.5 2.75v5.5c0 .138.112.25.25.25h1a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h3.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13 2a.25.25 0 0 0-.25-.25h-.5a.75.75 0 0 1 0-1.5h.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 14.25 12H14v1.543a1.458 1.458 0 0 1-2.487 1.03L9.22 12.28a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l2.22 2.22v-2.19a.75.75 0 0 1 .75-.75h1a.25.25 0 0 0 .25-.25Z"></path></svg>Premium Support</span><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--muted___rE6mh Primer_Brand__Text-module__Text--200____P1wy Primer_Brand__Text-module__Text--antialiased___TYoXS NavLink-module__subtitle__X4gkW">Enterprise-grade 24/7 support</span></span></a></li></ul></div></li></ul></div></div></li><li><a href="https://github.com/pricing" data-analytics-event="{&quot;action&quot;:&quot;pricing&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;pricing&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;pricing_link_pricing_navbar&quot;}" class="MarketingNavigation-module__navLink__hUomM">Pricing</a></li></ul></nav><div class="MarketingHeader-module__ctaContainer__tBmPz"><div class="HeaderSearch-module__searchSlot__oVOUS"><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderSearch-module__trigger__zsF9q" type="button" aria-haspopup="dialog" aria-expanded="false" aria-label="Search or jump to, type / to search" data-analytics-event="{&quot;action&quot;:&quot;searchbar&quot;,&quot;tag&quot;:&quot;input&quot;,&quot;context&quot;:&quot;global&quot;,&quot;location&quot;:&quot;navbar&quot;,&quot;label&quot;:&quot;searchbar_input_global_navbar&quot;}"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"><span class="HeaderSearch-module__content__kMpxU"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search HeaderSearch-module__icon__wcrHX" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg><span class="HeaderSearch-module__label__d1iWG">Search</span><kbd class="HeaderSearch-module__kbd__HNG0o" aria-hidden="true">/</kbd></span></span></span></button><div class="d-none"></div></div><div class="AuthCTAs-module__signInWrap__q2P60"><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq AuthCTAs-module__desktopActionGap__UZuXT AuthCTAs-module__hiddenBelowLg__BfKBw" href="/login?return_to=https%3A%2F%2Fgithub.com%2Faphyr%2Fdistsys-class%2Fblob%2Fmaster%2FREADME.markdown" data-analytics-event="{&quot;action&quot;:&quot;sign_in&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;auth_cta&quot;,&quot;location&quot;:&quot;header&quot;,&quot;label&quot;:&quot;sign_in_link_auth_cta_header&quot;}" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="b8e6ccc4f18d35e5477a4d151c3f567aa0a54465d1c785cf459c1ec94acfa70e"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH">Sign in</span></span></a></div><a class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--secondary___gHnw_ Primer_Brand__Button-module__Button--size-small___zQrEw AuthCTAs-module__cta__WpwQq" href="/signup?ref_cta=Sign+up&amp;ref_loc=header+logged+out&amp;ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow&amp;source=header-repo&amp;source_repo=aphyr%2Fdistsys-class" data-analytics-event="{&quot;action&quot;:&quot;sign_up&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;context&quot;:&quot;auth_cta&quot;,&quot;location&quot;:&quot;header&quot;,&quot;label&quot;:&quot;sign_up_link_auth_cta_header&quot;}" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="b8e6ccc4f18d35e5477a4d151c3f567aa0a54465d1c785cf459c1ec94acfa70e"><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-secondary___eJ0_a">Sign up</span></span></a><button class="Primer_Brand__Button-module__Button___scH9Z Primer_Brand__Button-module__Button--subtle___F7pEE Primer_Brand__Button-module__Button--size-small___zQrEw HeaderAppearanceSettings-module__trigger__hUheK" type="button" aria-haspopup="dialog" aria-labelledby="_R_fbd_"><span class="Primer_Brand__Button-module__Button__leading-visual___jjtTe" data-testid="Button-leading-visual"><svg data-component="Octicon" focusable="false" aria-hidden="true" class="octicon octicon-sliders Primer_Brand__Button-module__Button__icon-visual____qybb" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path></svg></span><span class="Primer_Brand__Button-module__Button__text___ED0bX"><span class="Primer_Brand__Text-module__Text___XeGJJ Primer_Brand__Text-module__Text-font--mona-sans___a8XJD Primer_Brand__Text-module__Text--default___GhPh_ Primer_Brand__Text-module__Text--100___B2ueX Primer_Brand__Text-module__Text--weight-medium___qJKf_ Primer_Brand__Button-module__Button--label___qrkyz Primer_Brand__Button-module__Button--label-subtle___8ndWH"></span></span></button><div class="Primer_Brand__Tooltip-module__Tooltip___0Eipx" data-direction="s" aria-hidden="true" id="_R_fbd_">Appearance settings</div></div></div></div></div><div class="MarketingHeader-module__bottomBorder__uZT38" aria-hidden="true"></div></header></div></div>
</react-partial>
<div hidden="hidden" data-view-component="true" class="js-stale-session-flash stale-session-flash flash flash-warn flash-full">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
<span class="js-stale-session-flash-signed-in" hidden>You signed in with another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
<span class="js-stale-session-flash-signed-out" hidden>You signed out in another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
<span class="js-stale-session-flash-switched" hidden>You switched accounts on another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
<button id="icon-button-da8d7534-9f90-4d38-83f8-67178a27c480" aria-labelledby="tooltip-5cbd787d-e439-43c8-ac57-13a01ba38934" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium flash-close js-flash-close"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x Button-visual">
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</button><tool-tip id="tooltip-5cbd787d-e439-43c8-ac57-13a01ba38934" for="icon-button-da8d7534-9f90-4d38-83f8-67178a27c480" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Dismiss alert</tool-tip>
</div>
</div>
<div id="start-of-content" class="show-on-focus"></div>
<div id="js-flash-container" class="flash-container" data-turbo-replace>
<template class="js-flash-template">
<div class="flash flash-full {{ className }}">
<div >
<button autofocus class="flash-close js-flash-close" type="button" aria-label="Dismiss this message">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</button>
<div aria-atomic="true" role="alert" class="js-flash-alert">
<div>{{ message }}</div>
</div>
</div>
</div>
</template>
</div>
<div
class="application-main "
data-commit-hovercards-enabled
data-discussion-hovercards-enabled
data-issue-and-pr-hovercards-enabled
data-project-hovercards-enabled
>
<div itemscope itemtype="http://schema.org/SoftwareSourceCode" class="">
<main id="js-repo-pjax-container" >
<div id="repository-container-header" class="tmp-pt-3 hide-full-screen" style="background-color: var(--page-header-bgColor, var(--color-page-header-bg));" data-turbo-replace>
<div class="d-flex flex-nowrap flex-justify-end tmp-mb-3 tmp-px-3 tmp-px-lg-5" style="gap: 1rem;">
<div class="flex-auto min-width-0 width-fit">
<div class=" d-flex flex-wrap flex-items-center wb-break-word f3 text-normal">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo color-fg-muted mr-2 tmp-mr-2">
<path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path>
</svg>
<span class="author flex-self-stretch" itemprop="author">
<a class="url fn" rel="author" data-hovercard-type="user" data-hovercard-url="/users/aphyr/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="/aphyr">
aphyr
</a> </span>
<span class="mx-1 flex-self-stretch color-fg-muted">/</span>
<strong itemprop="name" class="mr-2 flex-self-stretch">
<a data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" href="/aphyr/distsys-class">distsys-class</a>
</strong>
<span></span><span class="Label Label--secondary v-align-middle mr-1">Public</span>
</div>
</div>
<div id="repository-details-container" class="flex-shrink-0" data-turbo-replace style="max-width: 70%;">
<ul class="pagehead-actions flex-shrink-0 d-none d-md-inline" style="padding: 2px 0;">
<li>
<a href="/login?return_to=%2Faphyr%2Fdistsys-class" rel="nofollow" id="repository-details-watch-button" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;notification subscription menu watch&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="85f75acb98e8921450c6df50b14f90eaf073ee80c4d6bd9653e08ada72879c88" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2 tmp-mr-2">
<path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path>
</svg>Notifications
</a> <tool-tip id="tooltip-10d4d856-55e2-464a-942b-6dbf221c479a" for="repository-details-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute">You must be signed in to change notification settings</tool-tip>
</li>
<li>
<a icon="repo-forked" id="fork-button" href="/login?return_to=%2Faphyr%2Fdistsys-class" rel="nofollow" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;repo details fork button&quot;,&quot;repository_id&quot;:55260959,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="0f6cbbd23e63c93d19c5a64522746c7c33b9fcf5f5b6d6f7ddeb77c45b2edc2b" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2 tmp-mr-2">
<path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
</svg>Fork
<span id="repo-network-counter" data-pjax-replace="true" data-turbo-replace="true" title="718" data-view-component="true" class="Counter">718</span>
</a>
</li>
<li>
<div data-view-component="true" class="BtnGroup d-flex">
<a href="/login?return_to=%2Faphyr%2Fdistsys-class" rel="nofollow" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;star button&quot;,&quot;repository_id&quot;:55260959,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/aphyr/distsys-class/blob/master/README.markdown&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="42c4af65b59eb43ee997234915c358574d6108946cd3aae8971a7be4506d236f" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2 tmp-mr-2">
<path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
</svg><span data-view-component="true" class="d-inline">
Star
</span> <span id="repo-stars-counter-star" aria-label="9884 users starred this repository" data-singular-suffix="user starred this repository" data-plural-suffix="users starred this repository" data-turbo-replace="true" title="9,884" data-view-component="true" class="Counter js-social-count">9.9k</span>
</a></div>
</li>
</ul>
</div>
</div>
<div id="responsive-meta-container" data-turbo-replace>
</div>
<nav data-pjax="#js-repo-pjax-container" aria-label="Repository" data-view-component="true" class="js-repo-nav js-sidenav-container-pjax js-responsive-underlinenav overflow-hidden UnderlineNav px-3 tmp-px-3 px-md-4 tmp-px-md-4 px-lg-5 tmp-px-lg-5">
<ul data-view-component="true" class="UnderlineNav-body list-style-none">
<li data-view-component="true" class="d-inline-flex">
<a id="code-tab" href="/aphyr/distsys-class" data-tab-item="i0code-tab" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches repo_packages repo_deployments repo_attestations /aphyr/distsys-class" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g c" data-command-id="repositories:go-to-code" data-react-nav="code-view" data-react-nav-anchor="code-view-repo-link" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Code&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" aria-current="page" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item selected">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code UnderlineNav-octicon d-none d-sm-inline">
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
<span data-content="Code">Code</span>
<span id="code-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="issues-tab" href="/aphyr/distsys-class/issues" data-tab-item="i1issues-tab" data-selected-links="repo_issues repo_labels repo_milestones /aphyr/distsys-class/issues" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g i" data-command-id="repositories:go-to-issues" data-react-nav="issues-react" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Issues&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened UnderlineNav-octicon d-none d-sm-inline">
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
</svg>
<span data-content="Issues">Issues</span>
<span id="issues-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="pull-requests-tab" href="/aphyr/distsys-class/pulls" data-tab-item="i2pull-requests-tab" data-selected-links="repo_pulls checks /aphyr/distsys-class/pulls" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g p" data-command-id="repositories:go-to-pull-requests" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Pull requests&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request UnderlineNav-octicon d-none d-sm-inline">
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
</svg>
<span data-content="Pull requests">Pull requests</span>
<span id="pull-requests-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="6" data-view-component="true" class="Counter">6</span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="actions-tab" href="/aphyr/distsys-class/actions" data-tab-item="i3actions-tab" data-selected-links="repo_actions /aphyr/distsys-class/actions" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g a" data-command-id="repositories:go-to-actions" data-react-nav="actions-workflows" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Actions&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play UnderlineNav-octicon d-none d-sm-inline">
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
</svg>
<span data-content="Actions">Actions</span>
<span id="actions-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="projects-tab" href="/aphyr/distsys-class/projects" data-tab-item="i4projects-tab" data-selected-links="repo_projects new_repo_project repo_project /aphyr/distsys-class/projects" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g b" data-command-id="repositories:go-to-projects" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Projects&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table UnderlineNav-octicon d-none d-sm-inline">
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
</svg>
<span data-content="Projects">Projects</span>
<span id="projects-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="security-and-quality-tab" href="/aphyr/distsys-class/security" data-tab-item="i5security-and-quality-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /aphyr/distsys-class/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-command-id="repositories:go-to-security" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security and quality&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield UnderlineNav-octicon d-none d-sm-inline">
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
<span data-content="Security and quality">Security and quality</span>
<span id="security-and-quality-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>
</a></li>
<li data-view-component="true" class="d-inline-flex">
<a id="insights-tab" href="/aphyr/distsys-class/pulse" data-tab-item="i6insights-tab" data-selected-links="repo_graphs repo_contributors dependency_graph dependabot_updates pulse people community /aphyr/distsys-class/pulse" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-command-id="repositories:go-to-insights" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Insights&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph UnderlineNav-octicon d-none d-sm-inline">
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
<span data-content="Insights">Insights</span>
<span id="insights-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>
</a></li>
</ul>
<div style="visibility:hidden;" data-view-component="true" class="UnderlineNav-actions js-responsive-underlinenav-overflow position-absolute pr-3 tmp-pr-3 pr-md-4 tmp-pr-md-4 pr-lg-5 tmp-pr-lg-5 right-0"> <action-menu data-select-variant="none" data-view-component="true">
<focus-group direction="vertical" mnemonics retain>
<button id="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-button" popovertarget="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-overlay" aria-controls="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-list" aria-haspopup="true" aria-labelledby="tooltip-b92f50a0-4324-4a7c-b312-de055ce201ed" type="button" data-view-component="true" class="Button Button--iconOnly Button--secondary Button--medium UnderlineNav-item"> <svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-kebab-horizontal Button-visual">
<path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path>
</svg>
</button><tool-tip id="tooltip-b92f50a0-4324-4a7c-b312-de055ce201ed" for="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-button" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Additional navigation options</tool-tip>
<anchored-position data-target="action-menu.overlay" id="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-overlay" anchor="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-button" align="start" side="outside-bottom" anchor-offset="normal" popover="auto" data-view-component="true">
<div data-view-component="true" class="Overlay Overlay--size-auto">
<div data-view-component="true" class="Overlay-body Overlay-body--paddingNone"> <action-list>
<div data-view-component="true">
<ul aria-labelledby="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-button" id="action-menu-5f2c50c5-009f-469d-832d-c71b7977a43b-list" role="menu" data-view-component="true" class="ActionListWrap--inset ActionListWrap">
<li hidden="hidden" data-menu-item="i0code-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-52610375-91d7-46d1-ac08-1e2fe87dc0dc" href="/aphyr/distsys-class" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code">
<path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Code
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i1issues-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-bcb290e9-5c37-4214-b3f9-e51d587ffae4" href="/aphyr/distsys-class/issues" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened">
<path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Issues
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i2pull-requests-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-6aa59731-0a0f-48b2-b5e2-a093819fad69" href="/aphyr/distsys-class/pulls" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request">
<path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Pull requests
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i3actions-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-549f0ef4-577d-4832-ae3f-349d7fa8df46" href="/aphyr/distsys-class/actions" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play">
<path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Actions
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i4projects-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-3142afa0-bcd2-45d6-bde9-daf0bc95fdc8" href="/aphyr/distsys-class/projects" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table">
<path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Projects
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i5security-and-quality-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-0ee24aa3-3e82-4992-8a51-663f8b31c8c2" href="/aphyr/distsys-class/security" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield">
<path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Security and quality
</span>
</a>
</li>
<li hidden="hidden" data-menu-item="i6insights-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
<a tabindex="-1" id="item-7ad7ed9c-8296-4d4c-8bfb-2e1bd023057b" href="/aphyr/distsys-class/pulse" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
<span class="ActionListItem-visual ActionListItem-visual--leading">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph">
<path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
</span>
<span data-view-component="true" class="ActionListItem-label">
Insights
</span>
</a>
</li>
</ul>
</div></action-list>
</div>
</div></anchored-position> </focus-group>
</action-menu></div>
</nav>
</div>
<turbo-frame id="repo-content-turbo-frame" target="_top" data-turbo-action="advance" class="">
<div id="repo-content-pjax-container" class="repository-content " >
<react-app
app-name="code-view"
initial-path="/aphyr/distsys-class/blob/master/README.markdown"
style="display: block; min-height: calc(100vh - 64px);"
data-attempted-ssr="true"
data-ssr="true"
data-lazy="false"
data-alternate="false"
data-data-router-enabled="true"
data-react-profiling="false"
>
<script type="application/json" data-target="react-app.embeddedData">{"payload":{"codeViewBlobRoute":{"csv":null,"csvError":null,"headerInfo":{"toc":[{"level":1,"text":"An Introduction to Distributed Systems","anchor":"an-introduction-to-distributed-systems","htmlText":"An Introduction to Distributed Systems"},{"level":2,"text":"Intro","anchor":"intro","htmlText":"Intro"},{"level":2,"text":"What makes a thing distributed?","anchor":"what-makes-a-thing-distributed","htmlText":"What makes a thing distributed?"},{"level":2,"text":"Nodes and networks","anchor":"nodes-and-networks","htmlText":"Nodes and networks"},{"level":3,"text":"Nodes","anchor":"nodes","htmlText":"Nodes"},{"level":3,"text":"Networks as message flows","anchor":"networks-as-message-flows","htmlText":"Networks as message flows"},{"level":3,"text":"Causality diagrams","anchor":"causality-diagrams","htmlText":"Causality diagrams"},{"level":3,"text":"Synchronous networks","anchor":"synchronous-networks","htmlText":"Synchronous networks"},{"level":3,"text":"Semi-synchronous networks","anchor":"semi-synchronous-networks","htmlText":"Semi-synchronous networks"},{"level":3,"text":"Asynchronous networks","anchor":"asynchronous-networks","htmlText":"Asynchronous networks"},{"level":2,"text":"When networks go wrong","anchor":"when-networks-go-wrong","htmlText":"When networks go wrong"},{"level":2,"text":"Low level protocols","anchor":"low-level-protocols","htmlText":"Low level protocols"},{"level":3,"text":"TCP","anchor":"tcp","htmlText":"TCP"},{"level":3,"text":"UDP","anchor":"udp","htmlText":"UDP"},{"level":2,"text":"Clocks","anchor":"clocks","htmlText":"Clocks"},{"level":3,"text":"Wall Clocks","anchor":"wall-clocks","htmlText":"Wall Clocks"},{"level":3,"text":"Lamport Clocks","anchor":"lamport-clocks","htmlText":"Lamport Clocks"},{"level":3,"text":"Vector Clocks","anchor":"vector-clocks","htmlText":"Vector Clocks"},{"level":3,"text":"GPS \u0026 Atomic Clocks","anchor":"gps--atomic-clocks","htmlText":"GPS \u0026amp; Atomic Clocks"},{"level":2,"text":"Review","anchor":"review","htmlText":"Review"},{"level":2,"text":"Availability","anchor":"availability","htmlText":"Availability"},{"level":3,"text":"Total availability","anchor":"total-availability","htmlText":"Total availability"},{"level":3,"text":"Sticky availability","anchor":"sticky-availability","htmlText":"Sticky availability"},{"level":3,"text":"High availability","anchor":"high-availability","htmlText":"High availability"},{"level":3,"text":"Majority available","anchor":"majority-available","htmlText":"Majority available"},{"level":3,"text":"Quantifying availability","anchor":"quantifying-availability","htmlText":"Quantifying availability"},{"level":2,"text":"Consistency","anchor":"consistency","htmlText":"Consistency"},{"level":3,"text":"Monotonic Reads","anchor":"monotonic-reads","htmlText":"Monotonic Reads"},{"level":3,"text":"Monotonic Writes","anchor":"monotonic-writes","htmlText":"Monotonic Writes"},{"level":3,"text":"Read Your Writes","anchor":"read-your-writes","htmlText":"Read Your Writes"},{"level":3,"text":"Writes Follow Reads","anchor":"writes-follow-reads","htmlText":"Writes Follow Reads"},{"level":3,"text":"Causal consistency","anchor":"causal-consistency","htmlText":"Causal consistency"},{"level":3,"text":"Sequential consistency","anchor":"sequential-consistency","htmlText":"Sequential consistency"},{"level":3,"text":"Linearizability","anchor":"linearizability","htmlText":"Linearizability"},{"level":3,"text":"Transactional Models","anchor":"transactional-models","htmlText":"Transactional Models"},{"level":3,"text":"Does any of this actually matter?","anchor":"does-any-of-this-actually-matter","htmlText":"Does any of this actually matter?"},{"level":2,"text":"Tradeoffs","anchor":"tradeoffs","htmlText":"Tradeoffs"},{"level":3,"text":"Availability and Consistency","anchor":"availability-and-consistency","htmlText":"Availability and Consistency"},{"level":3,"text":"Harvest and Yield","anchor":"harvest-and-yield","htmlText":"Harvest and Yield"},{"level":3,"text":"Hybrid systems","anchor":"hybrid-systems","htmlText":"Hybrid systems"},{"level":3,"text":"Review","anchor":"review-1","htmlText":"Review"},{"level":2,"text":"Avoid Consensus Wherever Possible","anchor":"avoid-consensus-wherever-possible","htmlText":"Avoid Consensus Wherever Possible"},{"level":3,"text":"CALM conjecture","anchor":"calm-conjecture","htmlText":"CALM conjecture"},{"level":3,"text":"Gossip","anchor":"gossip","htmlText":"Gossip"},{"level":3,"text":"CRDTs","anchor":"crdts","htmlText":"CRDTs"},{"level":3,"text":"HATs","anchor":"hats","htmlText":"HATs"},{"level":2,"text":"Fine, We Need Consensus, What Now?","anchor":"fine-we-need-consensus-what-now","htmlText":"Fine, We Need Consensus, What Now?"},{"level":3,"text":"Paxos","anchor":"paxos","htmlText":"Paxos"},{"level":3,"text":"ZAB","anchor":"zab","htmlText":"ZAB"},{"level":3,"text":"Humming Consensus","anchor":"humming-consensus","htmlText":"Humming Consensus"},{"level":3,"text":"Viewstamped Replication","anchor":"viewstamped-replication","htmlText":"Viewstamped Replication"},{"level":3,"text":"Raft","anchor":"raft","htmlText":"Raft"},{"level":2,"text":"What About Transactions?","anchor":"what-about-transactions","htmlText":"What About Transactions?"},{"level":2,"text":"Review","anchor":"review-2","htmlText":"Review"},{"level":2,"text":"Characteristic latencies","anchor":"characteristic-latencies","htmlText":"Characteristic latencies"},{"level":3,"text":"Multicore systems","anchor":"multicore-systems","htmlText":"Multicore systems"},{"level":3,"text":"Local networks","anchor":"local-networks","htmlText":"Local networks"},{"level":3,"text":"Geographic replication","anchor":"geographic-replication","htmlText":"Geographic replication"},{"level":3,"text":"Review","anchor":"review-3","htmlText":"Review"},{"level":2,"text":"Common distributed systems","anchor":"common-distributed-systems","htmlText":"Common distributed systems"},{"level":3,"text":"Outsourced heaps","anchor":"outsourced-heaps","htmlText":"Outsourced heaps"},{"level":3,"text":"KV stores","anchor":"kv-stores","htmlText":"KV stores"},{"level":3,"text":"SQL databases","anchor":"sql-databases","htmlText":"SQL databases"},{"level":3,"text":"Search","anchor":"search","htmlText":"Search"},{"level":3,"text":"Coordination services","anchor":"coordination-services","htmlText":"Coordination services"},{"level":3,"text":"Streaming systems","anchor":"streaming-systems","htmlText":"Streaming systems"},{"level":3,"text":"Distributed queues","anchor":"distributed-queues","htmlText":"Distributed queues"},{"level":3,"text":"Review","anchor":"review-4","htmlText":"Review"},{"level":2,"text":"A Pattern Language","anchor":"a-pattern-language","htmlText":"A Pattern Language"},{"level":3,"text":"Don't distribute","anchor":"dont-distribute","htmlText":"Don't distribute"},{"level":3,"text":"Use an existing distributed system","anchor":"use-an-existing-distributed-system","htmlText":"Use an existing distributed system"},{"level":3,"text":"Never fail","anchor":"never-fail","htmlText":"Never fail"},{"level":3,"text":"Accept failure","anchor":"accept-failure","htmlText":"Accept failure"},{"level":3,"text":"Recovery First","anchor":"recovery-first","htmlText":"Recovery First"},{"level":3,"text":"Reconciliation Loops","anchor":"reconciliation-loops","htmlText":"Reconciliation Loops"},{"level":3,"text":"Backups","anchor":"backups","htmlText":"Backups"},{"level":3,"text":"Redundancy","anchor":"redundancy","htmlText":"Redundancy"},{"level":3,"text":"Sharding","anchor":"sharding","htmlText":"Sharding"},{"level":3,"text":"Independent domains","anchor":"independent-domains","htmlText":"Independent domains"},{"level":3,"text":"ID structure","anchor":"id-structure","htmlText":"ID structure"},{"level":3,"text":"Immutable values","anchor":"immutable-values","htmlText":"Immutable values"},{"level":3,"text":"Mutable identities","anchor":"mutable-identities","htmlText":"Mutable identities"},{"level":3,"text":"Confluence","anchor":"confluence","htmlText":"Confluence"},{"level":3,"text":"Backpressure","anchor":"backpressure","htmlText":"Backpressure"},{"level":3,"text":"Services for domain models","anchor":"services-for-domain-models","htmlText":"Services for domain models"},{"level":3,"text":"Structure Follows Social Spaces","anchor":"structure-follows-social-spaces","htmlText":"Structure Follows Social Spaces"},{"level":3,"text":"Cross-service coordination","anchor":"cross-service-coordination","htmlText":"Cross-service coordination"},{"level":3,"text":"Migrations","anchor":"migrations","htmlText":"Migrations"},{"level":3,"text":"Review","anchor":"review-5","htmlText":"Review"},{"level":2,"text":"Production Concerns","anchor":"production-concerns","htmlText":"Production Concerns"},{"level":3,"text":"Distributed systems are supported by your culture","anchor":"distributed-systems-are-supported-by-your-culture","htmlText":"Distributed systems are supported by your culture"},{"level":3,"text":"Test everything","anchor":"test-everything","htmlText":"Test everything"},{"level":3,"text":"\"It's Slow\"","anchor":"its-slow","htmlText":"\"It's Slow\""},{"level":3,"text":"Instrument everything","anchor":"instrument-everything","htmlText":"Instrument everything"},{"level":3,"text":"Logging","anchor":"logging","htmlText":"Logging"},{"level":3,"text":"Shadow traffic","anchor":"shadow-traffic","htmlText":"Shadow traffic"},{"level":3,"text":"Versioning","anchor":"versioning","htmlText":"Versioning"},{"level":3,"text":"Rollouts","anchor":"rollouts","htmlText":"Rollouts"},{"level":3,"text":"Automated Control","anchor":"automated-control","htmlText":"Automated Control"},{"level":3,"text":"Feature flags","anchor":"feature-flags","htmlText":"Feature flags"},{"level":3,"text":"Chaos engineering","anchor":"chaos-engineering","htmlText":"Chaos engineering"},{"level":3,"text":"Oh no, queues","anchor":"oh-no-queues","htmlText":"Oh no, queues"},{"level":2,"text":"Review","anchor":"review-6","htmlText":"Review"},{"level":2,"text":"Further reading","anchor":"further-reading","htmlText":"Further reading"},{"level":3,"text":"Online","anchor":"online","htmlText":"Online"},{"level":3,"text":"Trees","anchor":"trees","htmlText":"Trees"}]},"issueTemplate":null,"discussionTemplate":null,"richText":"\u003carticle class=\"markdown-body entry-content container-lg\" itemprop=\"text\"\u003e\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAn Introduction to Distributed Systems\u003c/h1\u003e\u003ca id=\"user-content-an-introduction-to-distributed-systems\" class=\"anchor\" aria-label=\"Permalink: An Introduction to Distributed Systems\" href=\"#an-introduction-to-distributed-systems\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eCopyright Kyle Kingsbury \u0026amp; Jepsen, LLC.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis outline accompanies a 16 to 32 hour \u003ca href=\"https://jepsen.io/training.html\" rel=\"nofollow\"\u003eoverview class on distributed systems\nfundamentals\u003c/a\u003e. The course aims to introduce\nsoftware engineers to the practical basics of distributed systems, through\nlecture and discussion, and optional\n\u003ca href=\"https://github.com/jepsen-io/maelstrom\"\u003elabwork\u003c/a\u003e. Participants will gain an\nintuitive understanding of key distributed systems terms, an overview of the\nalgorithmic landscape, and explore production concerns.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eIntro\u003c/h2\u003e\u003ca id=\"user-content-intro\" class=\"anchor\" aria-label=\"Permalink: Intro\" href=\"#intro\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMy name is Kyle Kingsbury\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePronouns: he/him\u003c/li\u003e\n\u003cli\u003eEmail: \u003ca href=\"mailto:aphyr@jepsen.io\"\u003eaphyr@jepsen.io\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eThis outline: \u003ca href=\"https://github.com/aphyr/distsys-class\"\u003ehttps://github.com/aphyr/distsys-class\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eI've worked on distributed systems from several perspectives\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e2003--2009: sysadmin, network operations, web developer\u003c/li\u003e\n\u003cli\u003e2009--2014: backend engineer, database wrangler at various web startups\u003c/li\u003e\n\u003cli\u003e2014--now: distributed systems safety testing (Jepsen)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThis class is aimed at practitioners\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"Things I Wish I Would Have Known\"\u003c/li\u003e\n\u003cli\u003eBackend engineers\u003c/li\u003e\n\u003cli\u003eFrontend engineers\u003c/li\u003e\n\u003cli\u003eOps folks\u003c/li\u003e\n\u003cli\u003eProduct managers\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThis class aims to prepare you to write, operate, and use distributed systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFirst half: theoretical framework\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA map of how concepts fit together\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNot about memorizing everything; it's about knowing where to look\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEstablish a shared language\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTo talk with your peers\u003c/li\u003e\n\u003cli\u003eTo read a paper\u003c/li\u003e\n\u003cli\u003eTo evaluate a system's claims\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUnderstand fundamental principles\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTime, ordering, nodes, networks, faults, liveness, safety\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eClasses of algorithms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat they can and can't do, when to apply each\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSecond half: practical concerns\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA grab bag of design patterns\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIllustrative anecdotes\u003c/li\u003e\n\u003cli\u003eComputers were, in fact, a mistake\u003c/li\u003e\n\u003cli\u003eWhat \u003cem\u003eisn't\u003c/em\u003e in the papers?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFinally, production concerns\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDeployment, debugging, monitoring\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eClass logistics\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis class is what you make of it\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe can go as deep as you want, or skip over familiar ground\u003c/li\u003e\n\u003cli\u003eJump in with questions, challenges, ideas at any time\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWe'll take ~10-minute breaks every hour or so, plus lunch\u003c/li\u003e\n\u003cli\u003eIf you need to duck out for whatever reason, that's fine!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOutages, kids, pets, whatever\u003c/li\u003e\n\u003cli\u003eThe outline on GitHub can help fill in the gaps\u003c/li\u003e\n\u003cli\u003eYou can ask me during class or a break about something you missed!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eI'll ask for your thoughts often, but you can also jump in whenever\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf you \u003cem\u003edon't\u003c/em\u003e want to be called on, that's cool--please let me know\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf you think of something after class, you can email me!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOr write it down for the next day's discussion\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRecording is prohibited\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTeaching is my livelihood\u003c/li\u003e\n\u003cli\u003eAnd this outline is free!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf we're doing labs:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThere are labs for this class!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMostly in the middle\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRemind folks of the prereq: \u003ca href=\"https://github.com/jepsen-io/maelstrom/blob/main/doc/01-getting-ready/index.md\"\u003eGetting Ready\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf we're remote:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou can turn off video whenever you like\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut please, if \u003cem\u003esome\u003c/em\u003e people can be on video, that's really helpful\u003c/li\u003e\n\u003cli\u003eSeeing faces helps me know whether the lecture is working for you!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFor the labs, we'll be doing a shared tmux session\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOK, let's get going!\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat makes a thing distributed?\u003c/h2\u003e\u003ca id=\"user-content-what-makes-a-thing-distributed\" class=\"anchor\" aria-label=\"Permalink: What makes a thing distributed?\" href=\"#what-makes-a-thing-distributed\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eLamport, 1987:\u003c/p\u003e\n\u003cblockquote\u003e\n\u003cp dir=\"auto\"\u003eA distributed system is one in which the failure of a computer\nyou didn't even know existed can render your own computer\nunusable.\u003c/p\u003e\n\u003c/blockquote\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFirst glance: *nix boxen in our colo, running processes communicating via\nTCP or UDP.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOr boxes in EC2, Rackspace, etc\u003c/li\u003e\n\u003cli\u003eMaybe communicating over InfiniBand\u003c/li\u003e\n\u003cli\u003eSeparated by inches and a LAN\u003c/li\u003e\n\u003cli\u003eOr by kilometers and the internet\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMost mobile apps are also taking part in a distributed system\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCommunicating over a truly awful network\u003c/li\u003e\n\u003cli\u003eSame goes for desktop web browsers\u003c/li\u003e\n\u003cli\u003eIt's not just servers--it's clients too!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMore generally: distributed systems are\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMade up of parts which interact\u003c/li\u003e\n\u003cli\u003eSlowly\u003c/li\u003e\n\u003cli\u003eAnd often unreliably\u003c/li\u003e\n\u003cli\u003eWhatever those mean for you\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSo also:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRedundant CPUs in an airplane\u003c/li\u003e\n\u003cli\u003eATMs and Point-of-Sale terminals\u003c/li\u003e\n\u003cli\u003eSpace probes\u003c/li\u003e\n\u003cli\u003ePaying bills\u003c/li\u003e\n\u003cli\u003eDoctors making referrals\u003c/li\u003e\n\u003cli\u003eTrying to make plans via text message\u003c/li\u003e\n\u003cli\u003eEvery business meeting ever\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eNodes and networks\u003c/h2\u003e\u003ca id=\"user-content-nodes-and-networks\" class=\"anchor\" aria-label=\"Permalink: Nodes and networks\" href=\"#nodes-and-networks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe call each part of a distributed system a \u003cem\u003enode\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAlso known as \u003cem\u003eprocesses\u003c/em\u003e, \u003cem\u003eagents\u003c/em\u003e, or \u003cem\u003eactors\u003c/em\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eNodes\u003c/h3\u003e\u003ca id=\"user-content-nodes\" class=\"anchor\" aria-label=\"Permalink: Nodes\" href=\"#nodes\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCharacteristic latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOperations inside a node are \"fast\"\u003c/li\u003e\n\u003cli\u003eOperations between nodes are \"slow\"\u003c/li\u003e\n\u003cli\u003eWhat's fast or slow depends on what the system does\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eNodes are reliable\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFail as a unit\u003c/li\u003e\n\u003cli\u003eYou know when problems occur\u003c/li\u003e\n\u003cli\u003eState is coherent\u003c/li\u003e\n\u003cli\u003eState transitions occur in a nice, orderly fashion\u003c/li\u003e\n\u003cli\u003eTypically modeled as some kind of single-threaded state machine\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eA node could \u003cem\u003eitself\u003c/em\u003e be a distributed system\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut so long as that system as a whole provides \"fast, coherent\"\noperations, we can treat it as a single node.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFormal models for processes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://dspace.mit.edu/handle/1721.1/6952\" rel=\"nofollow\"\u003eActor model\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://www.cs.cmu.edu/~crary/819-f09/Hoare78.pdf\" rel=\"nofollow\"\u003eCommunicating Sequential Processes\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://golem.ph.utexas.edu/category/2009/08/the_pi_calculus.html\" rel=\"nofollow\"\u003ePi-calculus\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"http://lucacardelli.name/Talks/1998-03-30%20Mobile%20Ambients%20%28ETAPS%29.pdf\" rel=\"nofollow\"\u003eAmbient calculus\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFormal models for node failure\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCrash-stop\u003c/li\u003e\n\u003cli\u003eCrash-recover\u003c/li\u003e\n\u003cli\u003eCrash-amnesia\u003c/li\u003e\n\u003cli\u003eByzantine\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eNetworks as message flows\u003c/h3\u003e\u003ca id=\"user-content-networks-as-message-flows\" class=\"anchor\" aria-label=\"Permalink: Networks as message flows\" href=\"#networks-as-message-flows\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNodes interact via a \u003cem\u003enetwork\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHumans interact via spoken words\u003c/li\u003e\n\u003cli\u003eParticles interact via fields\u003c/li\u003e\n\u003cli\u003eComputers interact via IP, or UDP, or SCTP, or ...\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWe model those interactions as discrete \u003cem\u003emessages\u003c/em\u003e sent between nodes\u003c/li\u003e\n\u003cli\u003eMessages take \u003cem\u003etime\u003c/em\u003e to propagate\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis is the \"slow\" part of the distributed system\u003c/li\u003e\n\u003cli\u003eWe call this \"latency\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMessages can often be lost\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis is another \"unreliable\" part of the distributed system\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eNetwork is rarely homogenous\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSome links slower/smaller/more-likely-to-fail than others\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCausality diagrams\u003c/h3\u003e\u003ca id=\"user-content-causality-diagrams\" class=\"anchor\" aria-label=\"Permalink: Causality diagrams\" href=\"#causality-diagrams\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe can represent the interaction of nodes and the network as a diagram\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTime flows left-to-right, or top-to-bottom\u003c/li\u003e\n\u003cli\u003eNodes are lines in the direction of time (because they stay in place)\u003c/li\u003e\n\u003cli\u003eMessages as slanted paths \u003cem\u003econnecting\u003c/em\u003e nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSynchronous networks\u003c/h3\u003e\u003ca id=\"user-content-synchronous-networks\" class=\"anchor\" aria-label=\"Permalink: Synchronous networks\" href=\"#synchronous-networks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNodes execute in lockstep: time between node steps is always 1.\u003c/li\u003e\n\u003cli\u003eMessage delay is bounded\u003c/li\u003e\n\u003cli\u003eEffectively a perfect global clock\u003c/li\u003e\n\u003cli\u003eEasy to prove stuff about\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou probably don't have one\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSemi-synchronous networks\u003c/h3\u003e\u003ca id=\"user-content-semi-synchronous-networks\" class=\"anchor\" aria-label=\"Permalink: Semi-synchronous networks\" href=\"#semi-synchronous-networks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLike synchronous, but the clock is only approximate, e.g. in [c, 1]\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAsynchronous networks\u003c/h3\u003e\u003ca id=\"user-content-asynchronous-networks\" class=\"anchor\" aria-label=\"Permalink: Asynchronous networks\" href=\"#asynchronous-networks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eExecute independently, whenever: step time is anywhere in [0, 1]\u003c/li\u003e\n\u003cli\u003eUnbounded message delays\u003c/li\u003e\n\u003cli\u003eNo global clocks\u003c/li\u003e\n\u003cli\u003eWeaker than semi- or synchronous networks\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eImplies certain algorithms can't be as efficient\u003c/li\u003e\n\u003cli\u003eImplies certain algorithms are \u003cem\u003eimpossible\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eSee e.g. Attiya \u0026amp; Mavronicolas, \"Efficiency of Semi-Synchronous vs\nAsynchronous Networks\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIP networks are definitely asynchronous\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut \u003cem\u003ein practice\u003c/em\u003e the really pathological stuff doesn't happen\u003c/li\u003e\n\u003cli\u003eMost networks recover in seconds to weeks, not \"never\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConversely, human timescales are on the orders of seconds to weeks\u003c/li\u003e\n\u003cli\u003eSo we can't pretend the problems don't exist\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhen networks go wrong\u003c/h2\u003e\u003ca id=\"user-content-when-networks-go-wrong\" class=\"anchor\" aria-label=\"Permalink: When networks go wrong\" href=\"#when-networks-go-wrong\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAsynchronous networks are allowed to\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDuplicate\u003c/li\u003e\n\u003cli\u003eDelay\u003c/li\u003e\n\u003cli\u003eDrop\u003c/li\u003e\n\u003cli\u003eReorder\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDrops and delays are indistinguishable\u003c/li\u003e\n\u003cli\u003eByzantine networks are allowed to mess with messages \u003cem\u003earbitrarily\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIncluding rewriting their content\u003c/li\u003e\n\u003cli\u003eThey mostly don't happen in real networks\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMostly\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://www.pagerduty.com/blog/the-discovery-of-apache-zookeepers-poison-packet/\" rel=\"nofollow\"\u003ehttps://www.pagerduty.com/blog/the-discovery-of-apache-zookeepers-poison-packet/\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19\" rel=\"nofollow\"\u003ehttps://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eLow level protocols\u003c/h2\u003e\u003ca id=\"user-content-low-level-protocols\" class=\"anchor\" aria-label=\"Permalink: Low level protocols\" href=\"#low-level-protocols\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTCP\u003c/h3\u003e\u003ca id=\"user-content-tcp\" class=\"anchor\" aria-label=\"Permalink: TCP\" href=\"#tcp\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTCP \u003cem\u003eworks\u003c/em\u003e. Use it.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNot perfect; you can go faster\u003c/li\u003e\n\u003cli\u003eBut you'll know when this is the case\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIn practice, TCP prevents duplicates and reorders in the context of a single\nTCP conn\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut you're probably gonna open more than one connection\u003c/li\u003e\n\u003cli\u003eIf for no other reason than TCP conns eventually fail\u003c/li\u003e\n\u003cli\u003eAnd when that happens, you'll either a.) have missed messages or b.) retry\u003c/li\u003e\n\u003cli\u003eYou can \u003cem\u003ereconstruct\u003c/em\u003e an ordering by encoding your own sequence numbers on\ntop of TCP\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eUDP\u003c/h3\u003e\u003ca id=\"user-content-udp\" class=\"anchor\" aria-label=\"Permalink: UDP\" href=\"#udp\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSame addressing rules as TCP, but no stream invariants\u003c/li\u003e\n\u003cli\u003eLots of people want UDP \"for speed\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDon't consider that routers and nodes can and will arbitrarily drop packets\u003c/li\u003e\n\u003cli\u003eDon't consider that their packets \u003cem\u003ewill\u003c/em\u003e be duplicated\u003c/li\u003e\n\u003cli\u003eAnd reordered\u003c/li\u003e\n\u003cli\u003e\"But at least it's unbiased right?\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWRONG!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThis causes all kinds of havoc in, say, metrics collection\u003c/li\u003e\n\u003cli\u003eAnd debugging it is \u003cem\u003ehard\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eTCP gives you flow control and repacks logical messages into packets\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou'll need to re-build flow-control and backpressure\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTLS over UDP is a thing, but tough\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUDP is really useful where TCP FSM overhead is prohibitive\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMemory pressure\u003c/li\u003e\n\u003cli\u003eLots of short-lived conns and socket reuse\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEspecially useful where best-effort delivery maps well to the system goals\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eVoice calls: people will apologize and repeat themselves\u003c/li\u003e\n\u003cli\u003eGames: stutters and lag, but catch up later\u003c/li\u003e\n\u003cli\u003eHigher-level protocols impose sanity on underlying chaos\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eClocks\u003c/h2\u003e\u003ca id=\"user-content-clocks\" class=\"anchor\" aria-label=\"Permalink: Clocks\" href=\"#clocks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhen a system is split into independent parts, we still want some kind of\n\u003cem\u003eorder\u003c/em\u003e for events\u003c/li\u003e\n\u003cli\u003eClocks help us order things: first this, THEN that\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWall Clocks\u003c/h3\u003e\u003ca id=\"user-content-wall-clocks\" class=\"anchor\" aria-label=\"Permalink: Wall Clocks\" href=\"#wall-clocks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIn theory, the operating system clock gives you a partial order on system events\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCaveat: NTP is probably not as good as you think\u003c/li\u003e\n\u003cli\u003eCaveat: Definitely not well-synced between nodes\u003c/li\u003e\n\u003cli\u003eCaveat: Hardware can drift\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMultiple reports of supermicro TSCs causing system clock to run 26ms/s fast\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://groups.google.com/forum/#!search/Supermicro$20SYS-1029UX-LL1-S16$20system$20clock$20running$20too$20fast/mechanical-sympathy/oG9vLZVYjVA/DU-T9QpBAgAJ\" rel=\"nofollow\"\u003ehttps://groups.google.com/forum/#!search/Supermicro$20SYS-1029UX-LL1-S16$20system$20clock$20running$20too$20fast/mechanical-sympathy/oG9vLZVYjVA/DU-T9QpBAgAJ\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCaveat: By \u003cem\u003ecenturies\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNTP might not care\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"http://rachelbythebay.com/w/2017/09/27/2153/\" rel=\"nofollow\"\u003ehttp://rachelbythebay.com/w/2017/09/27/2153/\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCaveat: NTP can still jump the clock backwards (default: delta \u0026gt; 128 ms)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://www.eecis.udel.edu/~mills/ntp/html/clock.html\" rel=\"nofollow\"\u003ehttps://www.eecis.udel.edu/~mills/ntp/html/clock.html\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCaveat: POSIX time is not monotonic by \u003cem\u003edefinition\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCloudflare 2017: Leap second at midnight UTC meant time flowed backwards\u003c/li\u003e\n\u003cli\u003eAt the time, Go didn't offer access to CLOCK_MONOTONIC\u003c/li\u003e\n\u003cli\u003eComputed a negative duration, then fed it to rand.int63n(), which paniced\u003c/li\u003e\n\u003cli\u003eCaused DNS resolutions to fail: 1% of HTTP requests affected for several hours\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://blog.cloudflare.com/how-and-why-the-leap-second-affected-cloudflare-dns/\" rel=\"nofollow\"\u003ehttps://blog.cloudflare.com/how-and-why-the-leap-second-affected-cloudflare-dns/\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCaveat: The timescales you want to measure may not be attainable\u003c/li\u003e\n\u003cli\u003eCaveat: Threads can sleep\u003c/li\u003e\n\u003cli\u003eCaveat: Runtimes can sleep\u003c/li\u003e\n\u003cli\u003eCaveat: OS's can sleep\u003c/li\u003e\n\u003cli\u003eCaveat: \"Hardware\" can sleep\u003c/li\u003e\n\u003cli\u003eCaveat: The hypervisor can lie to you\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBy 16+ seconds in a 15 minute period!?\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://gist.github.com/sandfox/32e749b5eac861c93f1bbeb8782ae8fd\"\u003ehttps://gist.github.com/sandfox/32e749b5eac861c93f1bbeb8782ae8fd\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eJust don't.\u003c/li\u003e\n\u003cli\u003eAt least OS monotonic clocks are monotonic, right?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eoh no: \u003ca href=\"https://github.com/rust-lang/rust/blob/eed12bcd0cb281979c4c9ed956b9e41fda2bfaeb/src/libstd/time.rs#L201-L232\"\u003ehttps://github.com/rust-lang/rust/blob/eed12bcd0cb281979c4c9ed956b9e41fda2bfaeb/src/libstd/time.rs#L201-L232\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eUse CLOCK_BOOTTIME, which accounts for VM pauses\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eLamport Clocks\u003c/h3\u003e\u003ca id=\"user-content-lamport-clocks\" class=\"anchor\" aria-label=\"Permalink: Lamport Clocks\" href=\"#lamport-clocks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLamport 1977: \"Time, Clocks, and the Ordering of Events in a Distributed System\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOne clock per process\u003c/li\u003e\n\u003cli\u003eIncrements monotonically with each state transition: \u003ccode\u003et' = t + 1\u003c/code\u003e\u003c/li\u003e\n\u003cli\u003eIncluded with every message sent\u003c/li\u003e\n\u003cli\u003e\u003ccode\u003et' = max(t, t_msg + 1)\u003c/code\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf we have a total ordering of processes, we can impose a total order on\nevents\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut that order could be pretty unintuitive\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eVector Clocks\u003c/h3\u003e\u003ca id=\"user-content-vector-clocks\" class=\"anchor\" aria-label=\"Permalink: Vector Clocks\" href=\"#vector-clocks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGeneralizes Lamport clocks to a vector of all process clocks\u003c/li\u003e\n\u003cli\u003e\u003ccode\u003et_i' = max(t_i, t_msg_i)\u003c/code\u003e\u003c/li\u003e\n\u003cli\u003eFor every operation, increment that process' clock in the vector\u003c/li\u003e\n\u003cli\u003eProvides a partial causal order\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA \u0026lt; B iff all A_i \u0026lt;= B_i, and at least one A_i \u0026lt; B_i\u003c/li\u003e\n\u003cli\u003eSpecifically, given a pair of events, we can determine causal relationships\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA in causal past of B implies A \u0026lt; B\u003c/li\u003e\n\u003cli\u003eB in causal past of A implies B \u0026lt; A\u003c/li\u003e\n\u003cli\u003eIndependent otherwise\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePragmatically: the past is shared; the present is independent\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOnly \"present\", independent states need to be preserved\u003c/li\u003e\n\u003cli\u003eAncestor states can be discarded\u003c/li\u003e\n\u003cli\u003eLets us garbage-collect the past\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eO(processes) in space\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRequires coordination for GC\u003c/li\u003e\n\u003cli\u003eOr sacrifice correctness and prune old vclock entries\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eVariants\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDotted Version Vectors - for client/server systems, orders \u003cem\u003emore\u003c/em\u003e events\u003c/li\u003e\n\u003cli\u003eInterval Tree Clocks - for when processes come and go\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eGPS \u0026amp; Atomic Clocks\u003c/h3\u003e\u003ca id=\"user-content-gps--atomic-clocks\" class=\"anchor\" aria-label=\"Permalink: GPS \u0026amp; Atomic Clocks\" href=\"#gps--atomic-clocks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMuch better than NTP\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGlobally distributed total orders on the scale of milliseconds\u003c/li\u003e\n\u003cli\u003ePromote an asynchronous network to a semi-synchronous one\u003c/li\u003e\n\u003cli\u003eUnlocks more efficient algorithms\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOnly people with this right now are Google\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSpanner: globally distributed strongly consistent transactions\u003c/li\u003e\n\u003cli\u003eAnd they're not sharing\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMore expensive than you'd like\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSeveral hundred per GPS receiver\u003c/li\u003e\n\u003cli\u003eAtomic clocks for local corroboration: $$$$?\u003c/li\u003e\n\u003cli\u003eProbably want multiple types of GPS clock\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://rachelbythebay.com/w/2015/09/07/noleap/\" rel=\"nofollow\"\u003ehttps://rachelbythebay.com/w/2015/09/07/noleap/\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConfusing vendor checkbox which applied UTC corrections to GPS time\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eI don't know who's doing it yet, but I'd bet datacenters in the\nfuture will offer dedicated HW interfaces for bounded-accuracy time.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h2\u003e\u003ca id=\"user-content-review\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe've covered the fundamental primitives of distributed systems. Nodes\nexchange messages through a network, and both nodes and networks can fail in\nvarious ways. Protocols like TCP and UDP give us primitive channels for\nprocesses to communicate, and we can order events using clocks. Now, we'll\ndiscuss some high-level \u003cem\u003eproperties\u003c/em\u003e of distributed systems.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAvailability\u003c/h2\u003e\u003ca id=\"user-content-availability\" class=\"anchor\" aria-label=\"Permalink: Availability\" href=\"#availability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAvailability is basically the fraction of attempted operations which succeed.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTotal availability\u003c/h3\u003e\u003ca id=\"user-content-total-availability\" class=\"anchor\" aria-label=\"Permalink: Total availability\" href=\"#total-availability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNaive: every operation succeeds\u003c/li\u003e\n\u003cli\u003eIn consistency lit: every operation on a non-failing node succeeds\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNothing you can do about the failing nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSticky availability\u003c/h3\u003e\u003ca id=\"user-content-sticky-availability\" class=\"anchor\" aria-label=\"Permalink: Sticky availability\" href=\"#sticky-availability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEvery operation against a non-failing node succeeds\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWith the constraint that clients always talk to the same nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHigh availability\u003c/h3\u003e\u003ca id=\"user-content-high-availability\" class=\"anchor\" aria-label=\"Permalink: High availability\" href=\"#high-availability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBetter than if the system \u003cem\u003eweren't\u003c/em\u003e distributed.\u003c/li\u003e\n\u003cli\u003ee.g. tolerant of up to f failures, but no more\u003c/li\u003e\n\u003cli\u003eMaybe some operations fail\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMajority available\u003c/h3\u003e\u003ca id=\"user-content-majority-available\" class=\"anchor\" aria-label=\"Permalink: Majority available\" href=\"#majority-available\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOperations succeed \u003cem\u003eif\u003c/em\u003e they occur on a node which can communicate\nwith a majority of the cluster\u003c/li\u003e\n\u003cli\u003eOperations against minority components may fail\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eQuantifying availability\u003c/h3\u003e\u003ca id=\"user-content-quantifying-availability\" class=\"anchor\" aria-label=\"Permalink: Quantifying availability\" href=\"#quantifying-availability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe talk a lot about \"uptime\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAre systems up if nobody uses them?\u003c/li\u003e\n\u003cli\u003eIs it worse to be down during peak hours?\u003c/li\u003e\n\u003cli\u003eCan measure \"fraction of requests satisfied during a time window\"\u003c/li\u003e\n\u003cli\u003eThen plot that fraction over windows at different times\u003c/li\u003e\n\u003cli\u003eTimescale affects reported uptime\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eApdex\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNot all successes are equal\u003c/li\u003e\n\u003cli\u003eClassify operations into \"OK\", \"meh\", and \"awful\"\u003c/li\u003e\n\u003cli\u003eApdex = P(OK) + P(meh)/2\u003c/li\u003e\n\u003cli\u003eAgain, can report on a yearly basis\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"We achieved 99.999 apdex for the year\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAnd on finer timescales!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"Apdex for the user service just dropped to 0.5; page ops!\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIdeally: integral of happiness delivered by your service?\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eConsistency\u003c/h2\u003e\u003ca id=\"user-content-consistency\" class=\"anchor\" aria-label=\"Permalink: Consistency\" href=\"#consistency\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA consistency model is the set of \"safe\" histories of events in the system\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMonotonic Reads\u003c/h3\u003e\u003ca id=\"user-content-monotonic-reads\" class=\"anchor\" aria-label=\"Permalink: Monotonic Reads\" href=\"#monotonic-reads\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOnce I read a value, any subsequent read will return that state or later values\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMonotonic Writes\u003c/h3\u003e\u003ca id=\"user-content-monotonic-writes\" class=\"anchor\" aria-label=\"Permalink: Monotonic Writes\" href=\"#monotonic-writes\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf I make a write, any subsequent writes I make will take place \u003cem\u003eafter\u003c/em\u003e the\nfirst write\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRead Your Writes\u003c/h3\u003e\u003ca id=\"user-content-read-your-writes\" class=\"anchor\" aria-label=\"Permalink: Read Your Writes\" href=\"#read-your-writes\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOnce I write a value, any subsequent read I perform will return that write\n(or later values)\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWrites Follow Reads\u003c/h3\u003e\u003ca id=\"user-content-writes-follow-reads\" class=\"anchor\" aria-label=\"Permalink: Writes Follow Reads\" href=\"#writes-follow-reads\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOnce I read a value, any subsequent write will take place after that read\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCausal consistency\u003c/h3\u003e\u003ca id=\"user-content-causal-consistency\" class=\"anchor\" aria-label=\"Permalink: Causal consistency\" href=\"#causal-consistency\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSuppose operations can be linked by a DAG of causal relationships\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA write that follows a read, for instance, is causally related\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAssuming the process didn't just throw away the read data\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOperations not linked in that DAG are \u003cem\u003econcurrent\u003c/em\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eConstraint: before a process can execute an operation, all its precursors\nmust have executed on that node\u003c/li\u003e\n\u003cli\u003eConcurrent ops can be freely reordered\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSequential consistency\u003c/h3\u003e\u003ca id=\"user-content-sequential-consistency\" class=\"anchor\" aria-label=\"Permalink: Sequential consistency\" href=\"#sequential-consistency\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLike causal consistency, constrains possible orders\u003c/li\u003e\n\u003cli\u003eAll operations appear to execute atomically\u003c/li\u003e\n\u003cli\u003eEvery process agrees on the order of operations\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOperations from a given process always occur in order\u003c/li\u003e\n\u003cli\u003eBut nodes can lag behind\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eLinearizability\u003c/h3\u003e\u003ca id=\"user-content-linearizability\" class=\"anchor\" aria-label=\"Permalink: Linearizability\" href=\"#linearizability\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAll operations appear to execute atomically\u003c/li\u003e\n\u003cli\u003eEvery process agrees on the order of operations\u003c/li\u003e\n\u003cli\u003eEvery operation appears to take place \u003cem\u003ebetween\u003c/em\u003e its invocation and completion\ntimes\u003c/li\u003e\n\u003cli\u003eReal-time, external constraints let us build very strong systems\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTransactional Models\u003c/h3\u003e\u003ca id=\"user-content-transactional-models\" class=\"anchor\" aria-label=\"Permalink: Transactional Models\" href=\"#transactional-models\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSerializable: all operations (transactions) appear to execute atomically\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIn some order\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNo constraints on what that order is\u003c/li\u003e\n\u003cli\u003ePerfectly okay to read from the past, for instance\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAdya 1999: Weak Consistency: A Generalized Theory and Optimistic\nImplementations for Distributed Transactions\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eObjects have a total version order\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ex0 \u0026lt;\u0026lt; x1 \u0026lt;\u0026lt; x2\u003c/li\u003e\n\u003cli\u003eThis is an abstract requirement; DB may not know what the version order is\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDependencies between transactions\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ewrite-read (wr): T1 writes x1, T2 reads x1\u003c/li\u003e\n\u003cli\u003ewrite-write (ww): T1 writes x1, T2 writes x2\u003c/li\u003e\n\u003cli\u003eread-write (rw): T1 reads x1, T2 writes x2\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePhenomena are (mostly) cycles made of these edges\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eG0 (write cycle): cycle in ww\u003c/li\u003e\n\u003cli\u003eG1a (aborted read): committed txn reads version written by aborted txn\u003c/li\u003e\n\u003cli\u003eG1b (intermediate read): read non-final write of x by some other txn\u003c/li\u003e\n\u003cli\u003eG1c (cyclic information flow): cycle in ww U wr\u003c/li\u003e\n\u003cli\u003eG2 (anti-dependency cycle): cycle in ww U wr U rw\u003c/li\u003e\n\u003cli\u003eG2-item: G2 without predicates\u003c/li\u003e\n\u003cli\u003eG-single: G2 with only one rw\u003c/li\u003e\n\u003cli\u003eetc. etc.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIsolation levels prevent these phemomena\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRead Uncommitted: no G0\u003c/li\u003e\n\u003cli\u003eRead Committed: no G0, G1\u003c/li\u003e\n\u003cli\u003eRepeatable Read: no G0, G1, G2-item\u003c/li\u003e\n\u003cli\u003eSI: No g0, G1, G-single\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIt's a little more subtle than this, but we generally don't have time\nfor G-nonadjacent\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSerializable: no G0, G1, G2\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan augment graphs with process or realtime edges\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eStrong X means \"X, plus order of txns in real time\"\u003c/li\u003e\n\u003cli\u003eStrong Session X means \"X, plus order of txns in each session\"\u003c/li\u003e\n\u003cli\u003ee.g. Strong Serializable: no cycle in ww U wr U rw U realtime\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAlternate route: ANSI SQL's ACID isolation levels are weird\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBasically codified the effects of existing vendor implementations\u003c/li\u003e\n\u003cli\u003eDefinitions in the spec are ambiguous\u003c/li\u003e\n\u003cli\u003eEach ANSI SQL isolation level prohibits a weird phenomenon\u003c/li\u003e\n\u003cli\u003eRead Uncommitted\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePrevents P0: \u003cem\u003edirty writes\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ew1(x) ... w2(x)\u003c/li\u003e\n\u003cli\u003eCan't write over another transaction's data until it commits\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan read data while a transaction is still modifying it\u003c/li\u003e\n\u003cli\u003eCan read data that will be rolled back\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRead Committed\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePrevents P1: \u003cem\u003edirty reads\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ew1(x) ... r2(x)\u003c/li\u003e\n\u003cli\u003eCan't read a transaction's uncommitted values\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRepeatable Read\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePrevents P2: \u003cem\u003efuzzy reads\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003er1(x) ... w2(x)\u003c/li\u003e\n\u003cli\u003eOnce a transaction reads a value, it won't change until the transaction\ncommits\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSerializable\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePrevents P3: \u003cem\u003ephantoms\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGiven some predicate P\u003c/li\u003e\n\u003cli\u003er1(P) ... w2(y in P)\u003c/li\u003e\n\u003cli\u003eOnce a transaction reads a set of elements satisfying a query, that\nset won't change until the transaction commits\u003c/li\u003e\n\u003cli\u003eNot just values, but \u003cem\u003ewhich values even would have participated\u003c/em\u003e.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCursor Stability\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTransactions have a set of cursors\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA cursor refers to an object being accessed by the transaction\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRead locks are held until cursor is removed, or commit\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAt commit time, cursor is upgraded to a writelock\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePrevents lost-update\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSnapshot Isolation\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTransactions always read from a snapshot of committed data, taken before\nthe transaction begins\u003c/li\u003e\n\u003cli\u003eCommit can only occur if no other committed transaction with an\noverlapping [start..commit] interval has written to any of the objects\n\u003cem\u003ewe\u003c/em\u003e wrote\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFirst-committer-wins\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDoes any of this actually matter?\u003c/h3\u003e\u003ca id=\"user-content-does-any-of-this-actually-matter\" class=\"anchor\" aria-label=\"Permalink: Does any of this actually matter?\" href=\"#does-any-of-this-actually-matter\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eReal world just isn't that concurrent\u003c/li\u003e\n\u003cli\u003ePlenty of companies get by on Read Committed\u003c/li\u003e\n\u003cli\u003eBut \u003cem\u003emalicious\u003c/em\u003e attackers can \u003cem\u003einduce\u003c/em\u003e concurrency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFlexcoin\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBitcoin exchange which allowed users to create money by shuffling between accounts\u003c/li\u003e\n\u003cli\u003eAttacked in 2014, 365,000 GBP stolen\u003c/li\u003e\n\u003cli\u003eExchange collapsed altogether\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePoloniex\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConcurrent withdrawals were improperly isolated, allowing users to overspend\u003c/li\u003e\n\u003cli\u003eSafety audits didn't notice negative balances\u003c/li\u003e\n\u003cli\u003e12.3% of exchange funds stolen; loss spread among users\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"http://www.bailis.org/papers/acidrain-sigmod2017.pdf\" rel=\"nofollow\"\u003eWarszawski \u0026amp; Bailis 2017: ACIDRain\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAutomated identification of consistency violation in web apps\u003c/li\u003e\n\u003cli\u003ee.g. Buy one gift card, then spend it an unlimited number of times\u003c/li\u003e\n\u003cli\u003ee.g. Buy a pen, add a laptop to cart during checkout, get a free laptop\u003c/li\u003e\n\u003cli\u003eVulnerabilities found in over 50% of all eCommerce web sites\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWeak DB isolation defaults\u003c/li\u003e\n\u003cli\u003eImproper use of transactional scope\u003c/li\u003e\n\u003cli\u003eFailing to use any transactions whatsoever\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://chadscira.com/post/5fa269d46142ac544e013d6e/DISCLOSURE-Unlimited-Chase-Ultimate-Rewards-Points\" rel=\"nofollow\"\u003eChase Bank's credit card rewards system\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConcurrent transfers between balances allowed the creation of $70,000 USD\nin travel vouchers.\u003c/li\u003e\n\u003cli\u003eRedeemable for cash!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTradeoffs\u003c/h2\u003e\u003ca id=\"user-content-tradeoffs\" class=\"anchor\" aria-label=\"Permalink: Tradeoffs\" href=\"#tradeoffs\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIdeally, we want total availability and linearizability\u003c/li\u003e\n\u003cli\u003eConsistency requires coordination\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf every order is allowed, we don't need to do any work!\u003c/li\u003e\n\u003cli\u003eIf we want to disallow some orders of events, we have to exchange messages\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCoordinating comes (generally) with costs\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMore consistency is slower\u003c/li\u003e\n\u003cli\u003eMore consistency is more intuitive\u003c/li\u003e\n\u003cli\u003eMore consistency is less available\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAvailability and Consistency\u003c/h3\u003e\u003ca id=\"user-content-availability-and-consistency\" class=\"anchor\" aria-label=\"Permalink: Availability and Consistency\" href=\"#availability-and-consistency\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCAP Theorem: Linearizability OR total availability\u003c/li\u003e\n\u003cli\u003eBut wait, there's more!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBailis 2014: Highly Available Transactions: Virtues and Limitations\u003c/li\u003e\n\u003cli\u003eOther theorems disallow totally or sticky available...\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eStrong serializable\u003c/li\u003e\n\u003cli\u003eSerializable\u003c/li\u003e\n\u003cli\u003eRepeatable Read\u003c/li\u003e\n\u003cli\u003eCursor Stability\u003c/li\u003e\n\u003cli\u003eSnapshot Isolation\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eYou can have \u003cem\u003esticky\u003c/em\u003e available...\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCausal\u003c/li\u003e\n\u003cli\u003ePRAM\u003c/li\u003e\n\u003cli\u003eRead Your Writes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eYou can have \u003cem\u003etotally\u003c/em\u003e available...\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRead Uncommitted\u003c/li\u003e\n\u003cli\u003eRead Committed\u003c/li\u003e\n\u003cli\u003eMonotonic Atomic View\u003c/li\u003e\n\u003cli\u003eWrites Follow Reads\u003c/li\u003e\n\u003cli\u003eMonotonic Reads\u003c/li\u003e\n\u003cli\u003eMonotonic Writes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHarvest and Yield\u003c/h3\u003e\u003ca id=\"user-content-harvest-and-yield\" class=\"anchor\" aria-label=\"Permalink: Harvest and Yield\" href=\"#harvest-and-yield\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFox \u0026amp; Brewer, 1999: Harvest, Yield, and Scalable Tolerant Systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYield: probability of completing a request\u003c/li\u003e\n\u003cli\u003eHarvest: fraction of data reflected in the response\u003c/li\u003e\n\u003cli\u003eExamples\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNode faults in a search engine can cause some results to go missing\u003c/li\u003e\n\u003cli\u003eUpdates may be reflected on some nodes but not others\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConsider an AP system split by a partition\u003c/li\u003e\n\u003cli\u003eYou can write data that some people can't read\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eStreaming video degrades to preserve low latency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThis is not an excuse to violate your safety invariants\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJust helps you quantify how much you can \u003cem\u003eexceed\u003c/em\u003e safety invariants\u003c/li\u003e\n\u003cli\u003ee.g. \"99% of the time, you can read 90% of your prior writes\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eStrongly dependent on workload, HW, topology, etc\u003c/li\u003e\n\u003cli\u003eCan tune harvest vs yield on a per-request basis\u003c/li\u003e\n\u003cli\u003e\"As much as possible in 10ms, please\"\u003c/li\u003e\n\u003cli\u003e\"I need everything, and I understand you might not be able to answer\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHybrid systems\u003c/h3\u003e\u003ca id=\"user-content-hybrid-systems\" class=\"anchor\" aria-label=\"Permalink: Hybrid systems\" href=\"#hybrid-systems\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSo, you've got a spectrum of choices!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChances are different parts of your infrastructure have different needs\u003c/li\u003e\n\u003cli\u003ePick the weakest model that meets your constraints\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut consider probabilistic bounds; visibility lag might be prohibitive\u003c/li\u003e\n\u003cli\u003eSee Probabilistically Bounded Staleness in Dynamo Quorums\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eNot all data is equal\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBig data is usually less important\u003c/li\u003e\n\u003cli\u003eSmall data is usually critical\u003c/li\u003e\n\u003cli\u003eLinearizable user ops, causally consistent social feeds\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h3\u003e\u003ca id=\"user-content-review-1\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-1\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eAvailability is a measure of how often operations succeed. Consistency models\nare the rules that govern what operations can happen and when. Stronger\nconsistency models generally come at the cost of performance and availability.\nNext, we'll talk about different ways to build systems, from weak to strong\nconsistency.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAvoid Consensus Wherever Possible\u003c/h2\u003e\u003ca id=\"user-content-avoid-consensus-wherever-possible\" class=\"anchor\" aria-label=\"Permalink: Avoid Consensus Wherever Possible\" href=\"#avoid-consensus-wherever-possible\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCALM conjecture\u003c/h3\u003e\u003ca id=\"user-content-calm-conjecture\" class=\"anchor\" aria-label=\"Permalink: CALM conjecture\" href=\"#calm-conjecture\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eConsistency As Logical Monotonicity\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf you can prove a system is logically monotonic, it is coordination free\u003c/li\u003e\n\u003cli\u003eWhat the heck is \"coordination\"\u003c/li\u003e\n\u003cli\u003eFor that matter, what's \"monotonic\"?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMonotonicity, informally, is retraction-free\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDeductions from partial information are never invalidated by new information\u003c/li\u003e\n\u003cli\u003eBoth relational algebra and Datalog without negation are monotone\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAmeloot, et al, 2011: Relational transducers for declarative networking\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTheorem which shows coordination-free networks of processes unaware of the\nnetwork extent can compute only monotone queries in Datalog\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis is not an easy read\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\"Coordination-free\" doesn't mean no communication\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAlgo succeeds even in face of arbitrary horizontal partitions\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIn very loose practical terms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTry to phrase your problem such that you only \u003cem\u003eadd\u003c/em\u003e new facts to the system\u003c/li\u003e\n\u003cli\u003eWhen you compute a new fact based on what's currently known, can you ensure\nthat fact will never be retracted?\u003c/li\u003e\n\u003cli\u003eConsider special \"sealing facts\" that mark a block of facts as complete\u003c/li\u003e\n\u003cli\u003eThese \"grow-only\" algorithms are usually easier to implement\u003c/li\u003e\n\u003cli\u003eLikely tradeoff: incomplete reads\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBloom language\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUnordered programming with flow analysis\u003c/li\u003e\n\u003cli\u003eCan tell you where coordination \u003cem\u003ewould\u003c/em\u003e be required\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eGossip\u003c/h3\u003e\u003ca id=\"user-content-gossip\" class=\"anchor\" aria-label=\"Permalink: Gossip\" href=\"#gossip\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMessage broadcast system\u003c/li\u003e\n\u003cli\u003eUseful for cluster management, service discovery, health, sensors, CDNs, etc\u003c/li\u003e\n\u003cli\u003eGenerally weak consistency / high availability\u003c/li\u003e\n\u003cli\u003eGlobal broadcast\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSend a message to every other node\u003c/li\u003e\n\u003cli\u003eO(nodes)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMesh networks\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEpidemic models\u003c/li\u003e\n\u003cli\u003eRelay to your neighbors\u003c/li\u003e\n\u003cli\u003ePropagation times on the order of max-free-path\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSpanning trees\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eInstead of a mesh, use a tree\u003c/li\u003e\n\u003cli\u003eHop up to a connector node which relays to other connector nodes\u003c/li\u003e\n\u003cli\u003eReduces superfluous messages\u003c/li\u003e\n\u003cli\u003eReduces latency\u003c/li\u003e\n\u003cli\u003ePlumtree (Leit ̃ao, Pereira, \u0026amp; Rodrigues, 2007: Epidemic Broadcast Trees)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePush-Sum\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKempe, Dobra, \u0026amp; Gehrke - Gossip-Based Computation of Aggregate Information\u003c/li\u003e\n\u003cli\u003eSum inputs from everyone you've received data from\u003c/li\u003e\n\u003cli\u003eBroadcast that to a random peer\u003c/li\u003e\n\u003cli\u003eExtensions for minima, maxima, means\u003c/li\u003e\n\u003cli\u003eHelpful for live metrics, rate limiting, routing, identifying cluster\nhotspots\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCRDTs\u003c/h3\u003e\u003ca id=\"user-content-crdts\" class=\"anchor\" aria-label=\"Permalink: CRDTs\" href=\"#crdts\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOrder-free datatypes that converge\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCounters, sets, maps, etc\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTolerate dupes, delays, and reorders\u003c/li\u003e\n\u003cli\u003eUnlike sequentially consistent systems, no \"single source of truth\"\u003c/li\u003e\n\u003cli\u003eBut unlike naive eventually consistent systems, never \u003cem\u003elose\u003c/em\u003e information\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUnless you explicitly make them lose information\u003c/li\u003e\n\u003cli\u003eWe call this property \"coalescence\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWorks well in highly-available systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWeb/mobile clients\u003c/li\u003e\n\u003cli\u003eDynamo\u003c/li\u003e\n\u003cli\u003eGossip\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eINRIA: Shapiro, Preguiça, Baquero, Zawirski, 2011: \"A comprehensive study of\nConvergent and Commutative Replicated Data Types\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eComposed of a data type X and a merge function m, which is:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAssociative: m(x1, m(x2, x3)) = m(m(x1, x2), x3)\u003c/li\u003e\n\u003cli\u003eCommutative: m(x1, x2) = m(x2, x1)\u003c/li\u003e\n\u003cli\u003eIdempotent: m(x1, x1) = m(x1)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEasy to build. Easy to reason about. Gets rid of all kinds of headaches.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDid communication fail? Just retry! It'll converge!\u003c/li\u003e\n\u003cli\u003eDid messages arrive out of order? It's fine!\u003c/li\u003e\n\u003cli\u003eHow do I synchronize two replicas? Just merge!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDownsides\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSome algorithms \u003cem\u003eneed\u003c/em\u003e order and can't be expressed with CRDTs\u003c/li\u003e\n\u003cli\u003eReads may be arbitrarily stale\u003c/li\u003e\n\u003cli\u003eHigher space costs\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHATs\u003c/h3\u003e\u003ca id=\"user-content-hats\" class=\"anchor\" aria-label=\"Permalink: HATs\" href=\"#hats\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBailis, Davidson, Fekete, et al, 2013: \"Highly Available Transactions,\nVirtues and Limitations\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGuaranteed responses from any replica\u003c/li\u003e\n\u003cli\u003eLow latency (1-3 orders of magnitude faster than serializable protocols!)\u003c/li\u003e\n\u003cli\u003eRead Committed\u003c/li\u003e\n\u003cli\u003eMonotonic Atomic View\u003c/li\u003e\n\u003cli\u003eExcellent for commutative/monotonic systems\u003c/li\u003e\n\u003cli\u003eForeign key constraints for multi-item updates\u003c/li\u003e\n\u003cli\u003eLimited uniqueness constraints\u003c/li\u003e\n\u003cli\u003eCan ensure convergence given arbitrary finite delay (\"eventual consistency\")\u003c/li\u003e\n\u003cli\u003eGood candidates for geographically distributed systems\u003c/li\u003e\n\u003cli\u003eProbably best in concert with stronger transactional systems\u003c/li\u003e\n\u003cli\u003eSee also: COPS, Swift, Eiger, Calvin, etc\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eFine, We Need Consensus, What Now?\u003c/h2\u003e\u003ca id=\"user-content-fine-we-need-consensus-what-now\" class=\"anchor\" aria-label=\"Permalink: Fine, We Need Consensus, What Now?\" href=\"#fine-we-need-consensus-what-now\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eThe consensus problem:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThree process types\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProposers: propose values\u003c/li\u003e\n\u003cli\u003eAcceptors: choose a value\u003c/li\u003e\n\u003cli\u003eLearners: read the chosen value\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eClasses of acceptors\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eN acceptors total\u003c/li\u003e\n\u003cli\u003eF acceptors allowed to fail\u003c/li\u003e\n\u003cli\u003eM malicious acceptors\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThree invariants:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNontriviality: Only values proposed can be learned\u003c/li\u003e\n\u003cli\u003eSafety: At most one value can be learned\u003c/li\u003e\n\u003cli\u003eLiveness: If a proposer p, a learner l, and a set of N-F acceptors are\nnon-faulty and can communicate with each other, and if p proposes a\nvalue, l will eventually learn a value.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eWhole classes of systems are \u003cem\u003eequivalent\u003c/em\u003e to the consensus problem\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSo any proofs we have here apply to those systems too\u003c/li\u003e\n\u003cli\u003eLock services\u003c/li\u003e\n\u003cli\u003eOrdered logs\u003c/li\u003e\n\u003cli\u003eReplicated state machines\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eFLP tells us consensus is impossible in asynchronous networks\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKill a process at the right time and you can break \u003cem\u003eany\u003c/em\u003e consensus algo\u003c/li\u003e\n\u003cli\u003eTrue but not as bad as you might think\u003c/li\u003e\n\u003cli\u003eRealistically, networks work \u003cem\u003eoften enough\u003c/em\u003e to reach consensus\u003c/li\u003e\n\u003cli\u003eMoreover, FLP assumes deterministic processes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eReal computers \u003cem\u003earen't\u003c/em\u003e deterministic\u003c/li\u003e\n\u003cli\u003eBen-Or 1983: \"Another Advantage of free choice\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNondeterministic algorithms \u003cem\u003ecan\u003c/em\u003e achieve consensus\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eLamport 2002: tight bounds for asynchronous consensus\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWith at least two proposers, or one malicious proposer, N \u0026gt; 2F + M\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"Need a majority\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWith at least 2 proposers, or one malicious proposer, it takes at least 2\nmessage delays to learn a proposal.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eThis is a pragmatically achievable bound\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIn stable clusters, you can get away with only a single round-trip to a\nmajority of nodes.\u003c/li\u003e\n\u003cli\u003eMore during cluster transitions.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePaxos\u003c/h3\u003e\u003ca id=\"user-content-paxos\" class=\"anchor\" aria-label=\"Permalink: Paxos\" href=\"#paxos\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePaxos is the Gold Standard of consensus algorithms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLamport 1989 - The Part Time Parliament\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWritten as a description of an imaginary Greek democracy\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eLamport 2001 - Paxos Made Simple\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"The Paxos algorithm for implementing a fault-tolerant distributed system\nhas been regarded as difficult to understand, perhaps because the\noriginal presentation was Greek to many readers [5]. In fact, it is among the\nsimplest and most obvious of distributed algorithms... The last section\nexplains the complete Paxos algorithm, which is obtained by the straightforward\napplication of consensus to the state machine approach for building a\ndistributed system—an approach that should be well-known, since it is the\nsubject of what is probably the most often-cited article on the theory of\ndistributed systems [4].\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eGoogle 2007 - Paxos Made Live\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNotes from productionizing Chubby, Google's lock service\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eVan Renesse 2011 - Paxos Made Moderately Complex\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTurns out you gotta optimize\u003c/li\u003e\n\u003cli\u003eAlso pseudocode would help\u003c/li\u003e\n\u003cli\u003eA page of pseudocode -\u0026gt; several thousand lines of C++\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eProvides consensus on independent proposals\u003c/li\u003e\n\u003cli\u003eTypically deployed in majority quorums, 5 or 7 nodes\u003c/li\u003e\n\u003cli\u003eSeveral optimizations\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMulti-Paxos\u003c/li\u003e\n\u003cli\u003eFast Paxos\u003c/li\u003e\n\u003cli\u003eGeneralized Paxos\u003c/li\u003e\n\u003cli\u003eIt's not always clear which of these optimizations to use, and which\ncan be safely combined\u003c/li\u003e\n\u003cli\u003eEach implementation uses a slightly different flavor\u003c/li\u003e\n\u003cli\u003ePaxos is really more of a \u003cem\u003efamily\u003c/em\u003e of algorithms than a well-described\nsingle entity\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUsed in a variety of production systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChubby\u003c/li\u003e\n\u003cli\u003eCassandra\u003c/li\u003e\n\u003cli\u003eRiak\u003c/li\u003e\n\u003cli\u003eFoundationDB\u003c/li\u003e\n\u003cli\u003eWANdisco SVN servers\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eNew research: Paxos quorums need not be majority: can optimize for fast phase-2 quorums \u003ca href=\"https://arxiv.org/abs/1608.06696\" rel=\"nofollow\"\u003eHoward, Malkhi, and Spiegelman\u003c/a\u003e.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe're not sure how to USE this yet\u003c/li\u003e\n\u003cli\u003eDurability still requires distribution\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eZAB\u003c/h3\u003e\u003ca id=\"user-content-zab\" class=\"anchor\" aria-label=\"Permalink: ZAB\" href=\"#zab\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZAB is the Zookeeper Atomic Broadcast protocol\u003c/li\u003e\n\u003cli\u003eJunqueira, Reed, and Serafini 2011 - Zab: High-performance broadcast for\nprimary-backup systems\u003c/li\u003e\n\u003cli\u003eDiffers from Paxos\u003c/li\u003e\n\u003cli\u003eProvides sequential consistency (linearizable writes, lagging ordered reads)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUseful because ZK clients typically want fast local reads\u003c/li\u003e\n\u003cli\u003eBut there's also a SYNC command that guarantees real-time visibility\u003c/li\u003e\n\u003cli\u003e(SYNC + op) allows linearizable reads as well\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAgain, majority quorum, 5 or 7 nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHumming Consensus\u003c/h3\u003e\u003ca id=\"user-content-humming-consensus\" class=\"anchor\" aria-label=\"Permalink: Humming Consensus\" href=\"#humming-consensus\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMetadata store for managing distributed system reconfiguration\u003c/li\u003e\n\u003cli\u003eLooks a little like CORFU's replicated log\u003c/li\u003e\n\u003cli\u003eSee also: chain replication\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eViewstamped Replication\u003c/h3\u003e\u003ca id=\"user-content-viewstamped-replication\" class=\"anchor\" aria-label=\"Permalink: Viewstamped Replication\" href=\"#viewstamped-replication\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePresented as a replication protocol, but also a consensus algorithm\u003c/li\u003e\n\u003cli\u003eTransaction processing plus a view change algorithm\u003c/li\u003e\n\u003cli\u003eMajority-known values are guaranteed to survive into the future\u003c/li\u003e\n\u003cli\u003eI'm not aware of any production systems, but I'm sure they're out there\u003c/li\u003e\n\u003cli\u003eAlong with Paxos, inspired Raft in some ways\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRaft\u003c/h3\u003e\u003ca id=\"user-content-raft\" class=\"anchor\" aria-label=\"Permalink: Raft\" href=\"#raft\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOngaro \u0026amp; Ousterhout 2014 - In Search of an Understandable Consensus Algorithm\u003c/li\u003e\n\u003cli\u003eLamport says it's easy, but we still have trouble grokking Paxos\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat if there were a consensus algorithm we could actually understand?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePaxos approaches independent decisions when what we \u003cem\u003ewant\u003c/em\u003e is state machines\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMaintains a replicated \u003cem\u003elog\u003c/em\u003e of state machine transitions instead\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAlso builds in cluster membership transitions, which is \u003cem\u003ekey\u003c/em\u003e for real systems\u003c/li\u003e\n\u003cli\u003eVery new, but we have a Coq proof of the core algorithm\u003c/li\u003e\n\u003cli\u003eCan be used to write arbitrary sequential or linearizable state machines\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRethinkDB\u003c/li\u003e\n\u003cli\u003eetcd\u003c/li\u003e\n\u003cli\u003eConsul\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat About Transactions?\u003c/h2\u003e\u003ca id=\"user-content-what-about-transactions\" class=\"anchor\" aria-label=\"Permalink: What About Transactions?\" href=\"#what-about-transactions\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIterated consensus gives us agreement on a single total order of operations\u003c/li\u003e\n\u003cli\u003eUnnecessary blocking betwixt transactions which \u003cem\u003ecould\u003c/em\u003e execute independently\u003c/li\u003e\n\u003cli\u003eHow do we improve performance?\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://aphyr.com/media/talks/2019/distributed-transaction-architectures.pdf\" rel=\"nofollow\"\u003eDistributed Transaction Architectures\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eSingle-writer\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAll updates go through a single queue, readers execute on snapshots\u003c/li\u003e\n\u003cli\u003eUsually involves some kind of persistent data structure\u003c/li\u003e\n\u003cli\u003eSerializable to strict-1SR\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://docs.datomic.com/on-prem/architecture.html\" rel=\"nofollow\"\u003eDatomic\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOK but multiple writers?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIn general, have several shards, each running a consensus-backed FSM\u003c/li\u003e\n\u003cli\u003eSome kind of protocol for cross-shard txns\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIndependent shards\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eA sort of halfway-step to general-purpose transactions\u003c/li\u003e\n\u003cli\u003eDisallow cross-shard txns\u003c/li\u003e\n\u003cli\u003eJust run a bunch of independent consensus FSMs\u003c/li\u003e\n\u003cli\u003eCan add a single global consensus group for cross-shard transactions\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLimited throughput though!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://docs.voltdb.com/UsingVoltDB/IntroHowVoltDBWorks.php\" rel=\"nofollow\"\u003eVoltDB\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf\" rel=\"nofollow\"\u003ePercolator\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSnapshot isolation over linearizable shards\u003c/li\u003e\n\u003cli\u003eTime Stamp Oracle assigns sequential txn timestamps (using consensus)\u003c/li\u003e\n\u003cli\u003eRead timestamp, read from leaders, prewrite, commit timestamp, commit, finalize\u003c/li\u003e\n\u003cli\u003e14 network hops, potentially all cross-DC\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://tikv.org/deep-dive/distributed-transaction/percolator/\" rel=\"nofollow\"\u003eTiDB\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://static.googleusercontent.com/media/research.google.com/en//archive/spanner-osdi2012.pdf\" rel=\"nofollow\"\u003eSpanner\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"External consistency\" (strict-1SR?)\u003c/li\u003e\n\u003cli\u003eSpeed up timestamp assignment by using GPS+Atomic clocks\u003c/li\u003e\n\u003cli\u003eBasically 2PC over Paxos groups\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLocks on Paxos leaders\u003c/li\u003e\n\u003cli\u003ePick one Paxos group to serve as the commit record for entire txn\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFixed latency floor to ensure timestamp monotonicity\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://blog.yugabyte.com/distributed-postgresql-on-a-google-spanner-architecture-storage-layer/\" rel=\"nofollow\"\u003eYugabyte DB\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/\" rel=\"nofollow\"\u003eCockroachDB\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf\" rel=\"nofollow\"\u003eCalvin\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOrder transactions in a log using consensus\u003c/li\u003e\n\u003cli\u003eShard log for arbitrarily high throughput\u003c/li\u003e\n\u003cli\u003ePeriodically seal log windows and apply transactions to shards\u003c/li\u003e\n\u003cli\u003eApplication requires no communication!\u003c/li\u003e\n\u003cli\u003eStrict-1SR\u003c/li\u003e\n\u003cli\u003e1 inter-DC round trip, more hops for local comms\u003c/li\u003e\n\u003cli\u003eScalable throughput\u003c/li\u003e\n\u003cli\u003eMinimum latency floor\u003c/li\u003e\n\u003cli\u003eTxns must be pure, expressed up-front\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCould be made interactive with extensions to protocol\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee \u003ca href=\"https://fauna.com/\" rel=\"nofollow\"\u003eFauna\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h2\u003e\u003ca id=\"user-content-review-2\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-2\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eSystems which only add facts, not retract them, require less coordination to\nbuild. We can use gossip systems to broadcast messages to other processes,\nCRDTs to merge updates from our peers, and HATs for weakly isolated\ntransactions. Serializability and linearizability require \u003cem\u003econsensus\u003c/em\u003e, which we\ncan obtain through Paxos, ZAB, VR, or Raft. Now, we'll talk about different\n\u003cem\u003escales\u003c/em\u003e of distributed systems.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCharacteristic latencies\u003c/h2\u003e\u003ca id=\"user-content-characteristic-latencies\" class=\"anchor\" aria-label=\"Permalink: Characteristic latencies\" href=\"#characteristic-latencies\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLatency is \u003cem\u003enever\u003c/em\u003e zero\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBandwidth goes up and up but we're bumping up against the physical limits\nof light and electrons\u003c/li\u003e\n\u003cli\u003eLatency budget shapes your system design\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHow many network calls can you afford?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDifferent kinds of systems have different definitions of \"slow\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDifferent goals\u003c/li\u003e\n\u003cli\u003eDifferent algorithms\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMulticore systems\u003c/h3\u003e\u003ca id=\"user-content-multicore-systems\" class=\"anchor\" aria-label=\"Permalink: Multicore systems\" href=\"#multicore-systems\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMulticore (and especially NUMA) architectures are sort of like a distributed system\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNodes don't fail pathologically, but message exchange is slow!\u003c/li\u003e\n\u003cli\u003eSynchronous network provided by a bus (e.g. Intel QPI)\u003c/li\u003e\n\u003cli\u003eWhole complicated set of protocols in HW \u0026amp; microcode to make memory look\nsane\u003c/li\u003e\n\u003cli\u003eNon-temporal store instructions (e.g. MOVNTI)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eThey provide abstractions to hide that distribution\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMFENCE/SFENCE/LFENCE\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIntroduce a serialization point against load/store instructions\u003c/li\u003e\n\u003cli\u003eCharacteristic latencies: ~100 cycles / ~30 ns\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eReally depends on HW, caches, instructions, etc\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCMPXCHG Compare-and-Swap (sequentially consistent modification of memory)\u003c/li\u003e\n\u003cli\u003eLOCK\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLock the full memory subsystem across cores!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBut those abstractions come with costs\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHardware lock elision may help but is nascent\u003c/li\u003e\n\u003cli\u003eBlog: Mechanical Sympathy\u003c/li\u003e\n\u003cli\u003eAvoid coordination between cores wherever possible\u003c/li\u003e\n\u003cli\u003eContext switches (process or thread!) can be expensive\u003c/li\u003e\n\u003cli\u003eProcessor pinning can really improve things\u003c/li\u003e\n\u003cli\u003eWhen writing multithreaded programs, try to divide your work into\nindependent chunks\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTry to align memory barriers to work unit boundaries\u003c/li\u003e\n\u003cli\u003eAllows the processor to cheat as much as possible within a work unit\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Danica Porobic, 2016: \u003ca href=\"https://infoscience.epfl.ch/record/219117/files/EPFL_TH7023.pdf\" rel=\"nofollow\"\u003eHigh Performance Transaction Processing on Non-Uniform Hardware Topologies\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eLocal networks\u003c/h3\u003e\u003ca id=\"user-content-local-networks\" class=\"anchor\" aria-label=\"Permalink: Local networks\" href=\"#local-networks\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou'll often deploy replicated systems across something like an ethernet LAN\u003c/li\u003e\n\u003cli\u003eMessage latencies can be as low as 100 micros\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut across any sizable network (EC2), expect low millis\u003c/li\u003e\n\u003cli\u003eSometimes, packets could be delayed by \u003cem\u003efive minutes\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003ePlan for this\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eNetwork is within an order of mag compared to uncached disk seeks\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOr faster, in EC2\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEC2 disk latencies can routinely hit 20ms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e200ms?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003cem\u003e20,000\u003c/em\u003e ms???\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBecause EBS is actually other computers\u003c/li\u003e\n\u003cli\u003eLMAO if you think anything in EC2 is real\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWait, \u003cem\u003ereal disks do this too\u003c/em\u003e?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat even are IO schedulers?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBut network is waaaay slower than memory/computation\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf your aim is \u003cem\u003ethroughput\u003c/em\u003e, work units should probably take longer than a\nmillisecond\u003c/li\u003e\n\u003cli\u003eBut there are other reasons to distribute\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSharding resources\u003c/li\u003e\n\u003cli\u003eIsolating failures\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eGeographic replication\u003c/h3\u003e\u003ca id=\"user-content-geographic-replication\" class=\"anchor\" aria-label=\"Permalink: Geographic replication\" href=\"#geographic-replication\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou deploy worldwide for two reasons\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEnd-user latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHumans can detect ~10ms lag, will tolerate ~100ms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSF--Denver: 50ms\u003c/li\u003e\n\u003cli\u003eSF--Tokyo: 100 ms\u003c/li\u003e\n\u003cli\u003eSF--Madrid: 200 ms\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOnly way to beat the speed of light: move the service closer\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDisaster recovery\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDatacenter power is good but not perfect\u003c/li\u003e\n\u003cli\u003eHurricanes are a thing\u003c/li\u003e\n\u003cli\u003eEntire Amazon regions can and will fail\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYes, regions, not AZs\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMinimum of 1 round-trip for consensus\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMaybe as bad as 4 rounds\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMaybe 4 rounds all the time if you have a bad Paxos impl (e.g. Cassandra)\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSo if you do Paxos between datacenters, be ready for that cost!\u003c/li\u003e\n\u003cli\u003eBecause the minimum latencies are higher than users will tolerate\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCache cache cache\u003c/li\u003e\n\u003cli\u003eQueue writes and relay asynchronously\u003c/li\u003e\n\u003cli\u003eConsider reduced consistency guarantees in exchange for lower latency\u003c/li\u003e\n\u003cli\u003eCRDTs can always give you safe local writes\u003c/li\u003e\n\u003cli\u003eCausal consistency and HATs can be good calls here\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWhat about strongly consistent stuff?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChances are a geographically distributed service has natural planes of\ncleavage\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEU users live on EU servers; US users live on US servers\u003c/li\u003e\n\u003cli\u003eUse consensus to migrate users between datacenters\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePin/proxy updates to home datacenter\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhich is hopefully the closest datacenter!\u003c/li\u003e\n\u003cli\u003eBut maybe not! I believe Facebook still pushes all writes through 1 DC!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWhere sequential consistency is OK, cache reads locally!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou probably leverage caching in a single DC already\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h3\u003e\u003ca id=\"user-content-review-3\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-3\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe discussed three characteristic scales for distributed systems: multicore\nprocessors coupled with a synchronous network, computers linked by a LAN, and\ndatacenters linked by the internet or dedicated fiber. CPU consequences are\nlargely performance concerns: knowing how to minimize coordination. On LANs,\nlatencies are short enough for many network hops before users take notice. In\ngeographically replicated systems, high latencies drive eventually consistent\nand datacenter-pinned solutions.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCommon distributed systems\u003c/h2\u003e\u003ca id=\"user-content-common-distributed-systems\" class=\"anchor\" aria-label=\"Permalink: Common distributed systems\" href=\"#common-distributed-systems\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eOutsourced heaps\u003c/h3\u003e\u003ca id=\"user-content-outsourced-heaps\" class=\"anchor\" aria-label=\"Permalink: Outsourced heaps\" href=\"#outsourced-heaps\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRedis, memcached, ...\u003c/li\u003e\n\u003cli\u003eData fits in memory, complex data structures\u003c/li\u003e\n\u003cli\u003eUseful when your language's built-in data structures are slow/awful\u003c/li\u003e\n\u003cli\u003eExcellent as a cache\u003c/li\u003e\n\u003cli\u003eOr as a quick-and-dirty scratchpad for shared state between platforms\u003c/li\u003e\n\u003cli\u003eNot particularly safe\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eKV stores\u003c/h3\u003e\u003ca id=\"user-content-kv-stores\" class=\"anchor\" aria-label=\"Permalink: KV stores\" href=\"#kv-stores\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRiak, Couch, Mongo, Cassandra, RethinkDB, HDFS, ...\u003c/li\u003e\n\u003cli\u003eOften 1,2,3 dimensions of keys\u003c/li\u003e\n\u003cli\u003eO(1) access, sometimes O(range) range scans by ID\u003c/li\u003e\n\u003cli\u003eNo strong relationships between values\u003c/li\u003e\n\u003cli\u003eObjects may be opaque or structured\u003c/li\u003e\n\u003cli\u003eLarge data sets\u003c/li\u003e\n\u003cli\u003eOften linear scalability\u003c/li\u003e\n\u003cli\u003eOften no transactions\u003c/li\u003e\n\u003cli\u003eRange of consistency models--often optional linearizable/sequential ops.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSQL databases\u003c/h3\u003e\u003ca id=\"user-content-sql-databases\" class=\"anchor\" aria-label=\"Permalink: SQL databases\" href=\"#sql-databases\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePostgres, MySQL, Percona XtraDB, Oracle, MSSQL, VoltDB, CockroachDB, ...\u003c/li\u003e\n\u003cli\u003eDefined by relational algebra: restrictions of products of records, etc\u003c/li\u003e\n\u003cli\u003eModerate sized data sets\u003c/li\u003e\n\u003cli\u003eAlmost always include multi-record transactions\u003c/li\u003e\n\u003cli\u003eRelations and transactions require coordination, which reduces scalability\u003c/li\u003e\n\u003cli\u003eMany systems are primary-secondary failover\u003c/li\u003e\n\u003cli\u003eAccess cost varies depending on indexes\u003c/li\u003e\n\u003cli\u003eTypically strong consistency (SI, serializable, strict serializable)\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSearch\u003c/h3\u003e\u003ca id=\"user-content-search\" class=\"anchor\" aria-label=\"Permalink: Search\" href=\"#search\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eElasticsearch, SolrCloud, ...\u003c/li\u003e\n\u003cli\u003eDocuments referenced by indices\u003c/li\u003e\n\u003cli\u003eModerate-to-large data sets\u003c/li\u003e\n\u003cli\u003eUsually O(1) document access, log-ish search\u003c/li\u003e\n\u003cli\u003eGood scalability\u003c/li\u003e\n\u003cli\u003eTypically weak consistency\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCoordination services\u003c/h3\u003e\u003ca id=\"user-content-coordination-services\" class=\"anchor\" aria-label=\"Permalink: Coordination services\" href=\"#coordination-services\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZookeeper, etcd, Consul, ...\u003c/li\u003e\n\u003cli\u003eTypically strong (sequential or linearizable) consistency\u003c/li\u003e\n\u003cli\u003eSmall data sets\u003c/li\u003e\n\u003cli\u003eUseful as a coordination primitive for stateless services\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eStreaming systems\u003c/h3\u003e\u003ca id=\"user-content-streaming-systems\" class=\"anchor\" aria-label=\"Permalink: Streaming systems\" href=\"#streaming-systems\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eStorm, Spark...\u003c/li\u003e\n\u003cli\u003eUsually custom-designed, or toolkits to build your own.\u003c/li\u003e\n\u003cli\u003eTypically small in-memory data volume\u003c/li\u003e\n\u003cli\u003eLow latencies\u003c/li\u003e\n\u003cli\u003eHigh throughput\u003c/li\u003e\n\u003cli\u003eWeak consistency\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDistributed queues\u003c/h3\u003e\u003ca id=\"user-content-distributed-queues\" class=\"anchor\" aria-label=\"Permalink: Distributed queues\" href=\"#distributed-queues\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKafka, Kestrel, Rabbit, IronMQ, ActiveMQ, HornetQ, Beanstalk, SQS, Celery, ...\u003c/li\u003e\n\u003cli\u003eJournals work to disk on multiple nodes for redundancy\u003c/li\u003e\n\u003cli\u003eUseful when you need to acknowledge work now, and actually do it later\u003c/li\u003e\n\u003cli\u003eSend data reliably between stateless services\u003c/li\u003e\n\u003cli\u003eThe \u003cem\u003eonly\u003c/em\u003e one I know that won't lose data in a partition is Kafka\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMaybe SQS?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues do not improve end-to-end latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAlways faster to do the work immediately\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues do not improve mean throughput\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMean throughput limited by consumers\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues do not provide total event ordering when consumers are concurrent\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYour consumers are almost definitely concurrent\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eLikewise, queues don't guarantee event order with async consumers\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBecause consumer side effects could take place out of order\u003c/li\u003e\n\u003cli\u003eSo, don't rely on order\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues can offer at-most-once or at-least-once delivery\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAnyone claiming otherwise is trying to sell you something\u003c/li\u003e\n\u003cli\u003eRecovering exactly-once delivery requires careful control of side effects\u003c/li\u003e\n\u003cli\u003eMake your queued operations idempotent\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues do improve burst throughput\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSmooth out load spikes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDistributed queues also improve fault tolerance (if they don't lose data)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf you don't need the fault-tolerance or large buffering, just use TCP\u003c/li\u003e\n\u003cli\u003eLots of people use a queue with six disk writes and fifteen network hops\nwhere a single socket write() could have sufficed\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues can get you out of a bind when you've chosen a poor runtime\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h3\u003e\u003ca id=\"user-content-review-4\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-4\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe use data structure stores as outsourced heaps: they're the duct tape of\ndistributed systems. KV stores and relational databases are commonly deployed\nas systems of record; KV stores use independent keys and are not well-suited to\nrelational data, but offer improved scalability and partial failure vs SQL\nstores, which offer rich queries and strong transactional guarantees.\nDistributed search and coordination services round out our basic toolkit for\nbuilding applications. Streaming systems are applied for continuous,\nlow-latency processing of datasets, and tend to look more like frameworks than\ndatabases. Their dual, distributed queues, focus on the \u003cem\u003emessages\u003c/em\u003e rather\nthan the \u003cem\u003etransformations\u003c/em\u003e.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eA Pattern Language\u003c/h2\u003e\u003ca id=\"user-content-a-pattern-language\" class=\"anchor\" aria-label=\"Permalink: A Pattern Language\" href=\"#a-pattern-language\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGeneral recommendations for building distributed systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHard-won experience\u003c/li\u003e\n\u003cli\u003eRepeating what other experts tell me\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOver beers\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHearsay\u003c/li\u003e\n\u003cli\u003eOversimplifications\u003c/li\u003e\n\u003cli\u003eCargo-culting\u003c/li\u003e\n\u003cli\u003eStuff I just made up\u003c/li\u003e\n\u003cli\u003eYMMV\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDon't distribute\u003c/h3\u003e\u003ca id=\"user-content-dont-distribute\" class=\"anchor\" aria-label=\"Permalink: Don't distribute\" href=\"#dont-distribute\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRule 1: don't distribute where you don't have to\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLocal systems have reliable primitives. Locks. Threads. Queues. Txns.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhen you move to a distributed system, you have to build from ground up.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIs this thing small enough to fit on one node?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"I have a big data problem\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSoftlayer will rent you a box with 3TB of ram for $5000/mo.\u003c/li\u003e\n\u003cli\u003eSupermicro will sell a 6TB box for ~$115,000 total.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eModern computers are FAST.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProduction JVM HTTP services I've known have pushed 50K requests/sec\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eParsing JSON events, journaling to disk, pushing to S3\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eProtocol buffers over TCP: 10 million events/sec\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e10-100 event batches/message, in-memory processing\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan this service tolerate a single node's guarantees?\u003c/li\u003e\n\u003cli\u003eCould we just stand up another one if it breaks?\u003c/li\u003e\n\u003cli\u003eCould manual intervention take the place of the distributed algorithm?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eUse an existing distributed system\u003c/h3\u003e\u003ca id=\"user-content-use-an-existing-distributed-system\" class=\"anchor\" aria-label=\"Permalink: Use an existing distributed system\" href=\"#use-an-existing-distributed-system\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf we have to distribute, can we push the work onto some other software?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat about a distributed database or log?\u003c/li\u003e\n\u003cli\u003eCan we pay Amazon to do this for us?\u003c/li\u003e\n\u003cli\u003eConversely, what are the care and feeding costs?\u003c/li\u003e\n\u003cli\u003eHow much do you have to learn to use/operate that distributed system?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eNever fail\u003c/h3\u003e\u003ca id=\"user-content-never-fail\" class=\"anchor\" aria-label=\"Permalink: Never fail\" href=\"#never-fail\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBuy really expensive hardware\u003c/li\u003e\n\u003cli\u003eMake changes to software and hardware in a controlled fashion\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDry-run deployments against staging environments\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePossible to build very reliable networks and machines\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAt the cost of moving slower, buying more expensive HW, finding talent\u003c/li\u003e\n\u003cli\u003eHW/network failure still \u003cem\u003ehappens\u003c/em\u003e, but sufficiently rare =\u0026gt; low priority\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAccept failure\u003c/h3\u003e\u003ca id=\"user-content-accept-failure\" class=\"anchor\" aria-label=\"Permalink: Accept failure\" href=\"#accept-failure\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDistributed systems aren't just characterized by \u003cem\u003elatency\u003c/em\u003e, but by\n\u003cem\u003erecurrent, partial failure\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eCan we accept this failure and move on with our lives?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat's our SLA anyway?\u003c/li\u003e\n\u003cli\u003eCan we recover by hand?\u003c/li\u003e\n\u003cli\u003eCan we pay someone to fix it?\u003c/li\u003e\n\u003cli\u003eCould insurance cover the damage?\u003c/li\u003e\n\u003cli\u003eCould we just call the customer and apologize?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSounds silly, but may be much cheaper\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe can never prevent 100% of system failures\u003c/li\u003e\n\u003cli\u003eConsciously choosing to recover \u003cem\u003eabove\u003c/em\u003e the level of the system\u003c/li\u003e\n\u003cli\u003eThis is how financial companies and retailers do it!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRecovery First\u003c/h3\u003e\u003ca id=\"user-content-recovery-first\" class=\"anchor\" aria-label=\"Permalink: Recovery First\" href=\"#recovery-first\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAssume a failure has just occurred: how will you recover?\u003c/li\u003e\n\u003cli\u003eMake this recovery the \u003cem\u003edefault\u003c/em\u003e path of execution\u003c/li\u003e\n\u003cli\u003eWriting recovery-first code keeps you from punting on error handling\u003c/li\u003e\n\u003cli\u003eExercising recovery code by default means you know it works\u003c/li\u003e\n\u003cli\u003eRecovery by default means you don't have to worry about different semantics\nduring a real fault\u003c/li\u003e\n\u003cli\u003eIf necessary, introduce a happy path for performance optimization\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut you lose some of these advantages!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReconciliation Loops\u003c/h3\u003e\u003ca id=\"user-content-reconciliation-loops\" class=\"anchor\" aria-label=\"Permalink: Reconciliation Loops\" href=\"#reconciliation-loops\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou've got a complex, stateful system, and want to move it somewhere\u003c/li\u003e\n\u003cli\u003eCould devise a plan of changes, and apply those changes in order\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut what if some change breaks? How do you recover?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eInstead, maintain a \u003cem\u003etarget\u003c/em\u003e: a representation of what you \u003cem\u003ewant\u003c/em\u003e the system\nto become\u003c/li\u003e\n\u003cli\u003eNext, write a function that looks at the current state, and \u003cem\u003ediffs\u003c/em\u003e it with\nthe target\u003c/li\u003e\n\u003cli\u003eUse that diff to find a step that moves the system closer to the target\u003c/li\u003e\n\u003cli\u003eRepeat indefinitely\u003c/li\u003e\n\u003cli\u003eRobust to faults and interference\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhat if your admin is tweaking things by hand?\u003c/li\u003e\n\u003cli\u003eWhat if two instances of the control system are running concurrently?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDeployed to great effect in systems like \u003ca href=\"https://queue.acm.org/detail.cfm?id=2898444\" rel=\"nofollow\"\u003eBorg \u0026amp; Kubernetes\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eAlso applicable to keeping data in sync between systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMaking sure every order is shipped \u0026amp; billed, for instance\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAgain, we're looking for \u003cem\u003emonotonicity\u003c/em\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBackups\u003c/h3\u003e\u003ca id=\"user-content-backups\" class=\"anchor\" aria-label=\"Permalink: Backups\" href=\"#backups\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBackups are essentially sequential consistency, BUT you lose a window of ops.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhen done correctly\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSome backup programs don't snapshot state, which leads to FS or DB\ncorruption\u003c/li\u003e\n\u003cli\u003eBroken fkey relationships, missing files, etc...\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAllow you to recover in a matter of minutes to days\u003c/li\u003e\n\u003cli\u003eBut more than fault recovery, they allow you to step back in time\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUseful for recovering from logical faults\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDistributed DB did its job correctly, but you told it to delete key\ndata\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRedundancy\u003c/h3\u003e\u003ca id=\"user-content-redundancy\" class=\"anchor\" aria-label=\"Permalink: Redundancy\" href=\"#redundancy\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOK, so failure is less of an option\u003c/li\u003e\n\u003cli\u003eWant to \u003cem\u003ereduce the probability of failure\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eHave the same state and same computation take place on several nodes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eI'm not a huge believer in active-spare\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSpare might have cold caches, broken disks, old versions, etc\u003c/li\u003e\n\u003cli\u003eSpares tend to fail when becoming active\u003c/li\u003e\n\u003cli\u003eActive-active wherever possible\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePredictability over efficiency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAlso not a huge fan of only having 2 copies\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNode failure probabilities just too high\u003c/li\u003e\n\u003cli\u003eOK for not-important data\u003c/li\u003e\n\u003cli\u003eI generally want three copies of data\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFor important stuff, 4 or 5\u003c/li\u003e\n\u003cli\u003eFor Paxos and other majority-quorum systems, odd numbers: 3, 5, 7\ncommon\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCommon DR strategy: Paxos across 5 nodes; 3 or 4 in primary DC\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOps can complete as soon as the local nodes ack; low latencies\u003c/li\u003e\n\u003cli\u003eResilient to single-node failure (though latencies will spike)\u003c/li\u003e\n\u003cli\u003eBut you still have a sequentially consistent backup in the other DC\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSo in the event you lose an entire DC, all's not lost\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Camille Fournier's talks on ZK deployment\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRedundancy improves availability so long as failures are uncorrelated\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFailures are not uncorrelated\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDisks from the same batch failing at the same time\u003c/li\u003e\n\u003cli\u003eSame-rack nodes failing when the top-of-rack switch blows\u003c/li\u003e\n\u003cli\u003eSame-DC nodes failing when the UPS blows\u003c/li\u003e\n\u003cli\u003eSee entire EC2 AZ failures\u003c/li\u003e\n\u003cli\u003eRunning the same bad computation on every node will break every node\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eExpensive queries\u003c/li\u003e\n\u003cli\u003eRiak list-keys\u003c/li\u003e\n\u003cli\u003eCassandra doomstones\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCascading failures\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThundering-herd\u003c/li\u003e\n\u003cli\u003eTCP incast\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eSharding\u003c/h3\u003e\u003ca id=\"user-content-sharding\" class=\"anchor\" aria-label=\"Permalink: Sharding\" href=\"#sharding\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe problem is too big\u003c/li\u003e\n\u003cli\u003eBreak the problem into parts small enough to fit on a node\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNot too small: small parts =\u0026gt; high overhead\u003c/li\u003e\n\u003cli\u003eNot too big: need to rebalance work units gradually from node to node\u003c/li\u003e\n\u003cli\u003eSomewhere around 10-100 work units/node is ideal, IMO\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIdeal: work units of equal size\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBeware hotspots\u003c/li\u003e\n\u003cli\u003eBeware changing workloads with time\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eKnow your bounds in advance\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHow big can a single part get before overwhelming a node?\u003c/li\u003e\n\u003cli\u003eHow do we enforce that limit \u003cem\u003ebefore\u003c/em\u003e it sinks a node in prod?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThen sinks all the other nodes, one by one, as the system rebalances\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAllocating shards to nodes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOften built in to DB\u003c/li\u003e\n\u003cli\u003eGood candidate for ZK, Etcd, and so on\u003c/li\u003e\n\u003cli\u003eSee Boundary's Ordasity\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eIndependent domains\u003c/h3\u003e\u003ca id=\"user-content-independent-domains\" class=\"anchor\" aria-label=\"Permalink: Independent domains\" href=\"#independent-domains\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSharding is a specific case of a more general pattern: avoiding coordination\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKeep as much independent as possible\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eImproves fault tolerance\u003c/li\u003e\n\u003cli\u003eImproves performance\u003c/li\u003e\n\u003cli\u003eReduces complexity\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSharding for scalability\u003c/li\u003e\n\u003cli\u003eAvoiding coordination via CRDTs\u003c/li\u003e\n\u003cli\u003eFlake IDs: \u003cem\u003emostly\u003c/em\u003e time-ordered identifiers, zero-coordination\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSee \u003ca href=\"http://yellerapp.com/posts/2015-02-09-flake-ids.html\" rel=\"nofollow\"\u003ehttp://yellerapp.com/posts/2015-02-09-flake-ids.html\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePartial availability: users can still use some parts of the system\u003c/li\u003e\n\u003cli\u003eProcessing a queue: more consumers reduces the impact of expensive events\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eID structure\u003c/h3\u003e\u003ca id=\"user-content-id-structure\" class=\"anchor\" aria-label=\"Permalink: ID structure\" href=\"#id-structure\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThings in our world have to have unique identifiers\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAt scale, ID structure can make or break you\u003c/li\u003e\n\u003cli\u003eConsider your access patterns\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eScans\u003c/li\u003e\n\u003cli\u003eSorts\u003c/li\u003e\n\u003cli\u003eShards\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSequential IDs require coordination: can you avoid them?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFlake IDs, UUIDs, ...\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFor \u003cem\u003eshardability\u003c/em\u003e, can your ID map directly to a shard?\u003c/li\u003e\n\u003cli\u003eSaaS app: object ID can also encode customer ID\u003c/li\u003e\n\u003cli\u003eTwitter: tweet ID can encode user ID\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eImmutable values\u003c/h3\u003e\u003ca id=\"user-content-immutable-values\" class=\"anchor\" aria-label=\"Permalink: Immutable values\" href=\"#immutable-values\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eData that never changes is trivial to store\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNever requires coordination\u003c/li\u003e\n\u003cli\u003eCheap replication and recovery\u003c/li\u003e\n\u003cli\u003eMinimal repacking on disk\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUseful for Cassandra, Riak, any LSM-tree DB.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOr for logs like Kafka!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEasy to reason about: either present or it's not\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEliminates all kinds of transactional headaches\u003c/li\u003e\n\u003cli\u003eExtremely cachable\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eExtremely high availability and durability, tunable write latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLow read latencies: can respond from closest replica\u003c/li\u003e\n\u003cli\u003eEspecially valuable for geographic distribution\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRequires garbage collection!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut there are good ways to do this\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMutable identities\u003c/h3\u003e\u003ca id=\"user-content-mutable-identities\" class=\"anchor\" aria-label=\"Permalink: Mutable identities\" href=\"#mutable-identities\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePointers to immutable values\u003c/li\u003e\n\u003cli\u003ePointers are small! Only metadata!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCan fit huge numbers of pointers on a small DB\u003c/li\u003e\n\u003cli\u003eGood candidate for consensus services or relational DBs\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAnd typically, not many pointers in the system\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYour entire DB could be represented by a single pointer\u003c/li\u003e\n\u003cli\u003eDatomic only has ~5 identities\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eStrongly consistent operations over identities can be \u003cem\u003ebacked\u003c/em\u003e by immutable\nHA storage\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTake advantage of AP storage latencies and scale\u003c/li\u003e\n\u003cli\u003eTake advantage of strong consistency over small datasets provided by\nconsensus systems\u003c/li\u003e\n\u003cli\u003eWrite availability limited by identity store\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut, reads eminently cachable if you only need sequential consistency\u003c/li\u003e\n\u003cli\u003eCan be even cheaper if you only need serializability\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Rich Hickey's talks on Datomic architecture\u003c/li\u003e\n\u003cli\u003eSee Pat Helland's 2013 RICON West keynote on Salesforce's storage\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eConfluence\u003c/h3\u003e\u003ca id=\"user-content-confluence\" class=\"anchor\" aria-label=\"Permalink: Confluence\" href=\"#confluence\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSystems which are order-independent are easier to construct and reason about\u003c/li\u003e\n\u003cli\u003eAlso helps us avoid coordination\u003c/li\u003e\n\u003cli\u003eCRDTs are confluent, which means we can apply updates without waiting\u003c/li\u003e\n\u003cli\u003eImmutable values are trivially confluent: once present, fixed\u003c/li\u003e\n\u003cli\u003eStreaming systems can leverage confluence as well:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBuffer events, and compute+flush when you know you've seen everything\u003c/li\u003e\n\u003cli\u003eEmit partial results so you can take action now, e.g. for monitoring\u003c/li\u003e\n\u003cli\u003eWhen full data is available, merge with + or max\u003c/li\u003e\n\u003cli\u003eBank ledgers are (mostly) confluent: txn order doesn't affect balance\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut when you need to enforce a minimum balance, no longer confluent\u003c/li\u003e\n\u003cli\u003eCombine with a sealing event (e.g. the day's end) to recover confluence\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Aiken, Widom, \u0026amp; Hellerstein 1992, \"Behavior of Database Production Rules\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBackpressure\u003c/h3\u003e\u003ca id=\"user-content-backpressure\" class=\"anchor\" aria-label=\"Permalink: Backpressure\" href=\"#backpressure\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eServices which talk to each other are usually connected by \u003cem\u003equeues\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eService and queue capacity is finite\u003c/li\u003e\n\u003cli\u003eHow do you handle it when a downstream service is unable to handle load?\n\u003col dir=\"auto\"\u003e\n\u003cli\u003eConsume resources and explode\u003c/li\u003e\n\u003cli\u003eShed load. Start dropping requests.\u003c/li\u003e\n\u003cli\u003eReject requests. Ignore the work and tell clients it failed.\u003c/li\u003e\n\u003cli\u003eApply backpressure to clients, asking them to slow down.\u003c/li\u003e\n\u003c/ol\u003e\n\u003c/li\u003e\n\u003cli\u003e2-4 allow the system to catch up and recover\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut backpressure reduces the volume of work that has to be retried\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBackpressure defers choice to producers: compositional\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eClients of load-shedding systems are locked into load-shedding\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThey have no way to tell that the system is hosed\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eClients of backpressure systems can apply backpressure to \u003cem\u003etheir clients\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOr shed load, if they choose\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf you're making an asynchronous system, \u003cem\u003ealways\u003c/em\u003e include backpressure\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYour users will thank you later\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFundamentally: \u003cem\u003ebounding resources\u003c/em\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRequest timeouts (bounded time)\u003c/li\u003e\n\u003cli\u003eExponential backoffs (bounded use)\u003c/li\u003e\n\u003cli\u003eBounded queues\u003c/li\u003e\n\u003cli\u003eBounded concurrency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Zach Tellman, \"Everything Will Flow\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eServices for domain models\u003c/h3\u003e\u003ca id=\"user-content-services-for-domain-models\" class=\"anchor\" aria-label=\"Permalink: Services for domain models\" href=\"#services-for-domain-models\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe problem is composed of interacting logical pieces\u003c/li\u003e\n\u003cli\u003ePieces have distinct code, performance, storage needs\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMonolithic applications are essentially \u003cem\u003emultitenant\u003c/em\u003e systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMultitenancy is tough\u003c/li\u003e\n\u003cli\u003eBut its often okay to run multiple logical \"services\" in the same process\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDivide your system into logical services for discrete parts of the domain\nmodel\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOO approach: each \u003cem\u003enoun\u003c/em\u003e is a service\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUser service\u003c/li\u003e\n\u003cli\u003eVideo service\u003c/li\u003e\n\u003cli\u003eIndex service\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eFunctional approach: each \u003cem\u003everb\u003c/em\u003e is a service\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAuth service\u003c/li\u003e\n\u003cli\u003eSearch service\u003c/li\u003e\n\u003cli\u003eDispatch/routing service\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eMost big systems I know of use a hybrid\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eServices for nouns is a good way to enforce \u003cem\u003edatatype invariants\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eServices for verbs is a good way to enforce \u003cem\u003etransformation invariants\u003c/em\u003e\u003c/li\u003e\n\u003cli\u003eSo have a basic User service, which is used \u003cem\u003eby\u003c/em\u003e an Auth service\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWhere you draw the line... well that's tricky\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eServices come with overhead: have as few as possible\u003c/li\u003e\n\u003cli\u003eConsider work units\u003c/li\u003e\n\u003cli\u003eSeparate services which need to scale independently\u003c/li\u003e\n\u003cli\u003eColocate services with tight dependencies and tight latency budgets\u003c/li\u003e\n\u003cli\u003eColocate services which use complementary resources (e.g. disk and CPU)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBy hand: Run memcache on rendering nodes\u003c/li\u003e\n\u003cli\u003eNewer shops: Google Borg, Mesos, Kubernetes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eServices should encapsulate and abstract\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTry to build trees instead of webs\u003c/li\u003e\n\u003cli\u003eAvoid having outsiders manipulate a service's data store directly\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eStructure Follows Social Spaces\u003c/h3\u003e\u003ca id=\"user-content-structure-follows-social-spaces\" class=\"anchor\" aria-label=\"Permalink: Structure Follows Social Spaces\" href=\"#structure-follows-social-spaces\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProduction software is a fundamentally social artifact\u003c/li\u003e\n\u003cli\u003eNatural alignment: a team or person owns a specific service\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJo Freeman, \"The Tyranny of Structurelessness\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://www.jofreeman.com/joreen/tyranny.htm\" rel=\"nofollow\"\u003ehttps://www.jofreeman.com/joreen/tyranny.htm\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eResponsibility and power should be explicit\u003c/li\u003e\n\u003cli\u003eRotate people through roles to prevent fiefdoms\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePeople own \u003cem\u003eproblem spaces\u003c/em\u003e, not services\u003c/li\u003e\n\u003cli\u003ePromotes information sharing\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBut don't rotate too often\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRamp-up costs in software are very high\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003ePair Freeman with Downs' \"Inside Bureaucracy\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://www.rand.org/content/dam/rand/pubs/papers/2008/P2963.pdf\" rel=\"nofollow\"\u003ehttps://www.rand.org/content/dam/rand/pubs/papers/2008/P2963.pdf\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAs the team grows, its mission and thinking will formalize\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSo too will services and their boundaries\u003c/li\u003e\n\u003cli\u003eGradually accruing body of assumptions about service relation to the world\u003c/li\u003e\n\u003cli\u003ePunctuated by rewrites to respond to changing external pressures\u003c/li\u003e\n\u003cli\u003eTushman \u0026amp; Romanelli, 1985: Organizational Transformation as Punctuated Equilibrium\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eServices can be libraries\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eInitially, \u003cem\u003eall\u003c/em\u003e your services should be libraries\u003c/li\u003e\n\u003cli\u003ePerfectly OK to depend on a user library in multiple services\u003c/li\u003e\n\u003cli\u003eLibraries with well-defined boundaries are easy to extract into\nservices later\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSocial structure governs the library/service boundary\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWith few users of a library, or tightly-coordinated users, changes are easy\u003c/li\u003e\n\u003cli\u003eBut across many teams, users have varying priorities and must be convinced\u003c/li\u003e\n\u003cli\u003eWhy should users do work to upgrade to a new library version?\u003c/li\u003e\n\u003cli\u003eServices \u003cem\u003eforce\u003c/em\u003e coordination through a defined API deprecation lifecycle\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou can also enforce this with libraries through code review \u0026amp; tooling\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eServices enable centralized control\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYour performance improvements affect everyone instantly\u003c/li\u003e\n\u003cli\u003eGradually shift to a new on-disk format or backing database\u003c/li\u003e\n\u003cli\u003eInstrument use of the service in one place\u003c/li\u003e\n\u003cli\u003eHarder to do these things with libraries\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eServices have costs\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe failure-complexity and latency overhead of a network call\u003c/li\u003e\n\u003cli\u003eTangled food web of service dependencies\u003c/li\u003e\n\u003cli\u003eHard to statically analyze codepaths\u003c/li\u003e\n\u003cli\u003eYou thought library API versioning was hard\u003c/li\u003e\n\u003cli\u003eAdditional instrumentation/deployment\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eServices can use good client libraries\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThat library might be \"Open a socket\" or an HTTP client\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLeverage HTTP headers!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAccept headers for versioning\u003c/li\u003e\n\u003cli\u003eLots of support for caching and proxying\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHaproxy is an excellent router for both HTTP and TCP services\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEventually, library might include mock IO\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eService team is responsible for testing that the service provides an API\u003c/li\u003e\n\u003cli\u003eWhen the API is known to be stable, every client can \u003cem\u003eassume\u003c/em\u003e it works\u003c/li\u003e\n\u003cli\u003eRemoves the need for network calls in test suites\u003c/li\u003e\n\u003cli\u003eDramatic reduction in test runtime and dev environment complexity\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eCross-service coordination\u003c/h3\u003e\u003ca id=\"user-content-cross-service-coordination\" class=\"anchor\" aria-label=\"Permalink: Cross-service coordination\" href=\"#cross-service-coordination\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCoordination between services requires special protocols\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHave to re-invent transactions\u003c/li\u003e\n\u003cli\u003eGo commutative where possible\u003c/li\u003e\n\u003cli\u003eSagas\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWas written for a single-node world: we have to be clever in distributed contexts\u003c/li\u003e\n\u003cli\u003eTransactions must be idempotent, OR commute with rollbacks\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"http://www.cs.ucsb.edu/~vaibhavarora/Typhon-Ieee-Cloud-2017.pdf\" rel=\"nofollow\"\u003eTyphon/Cerberus\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProtocols for causal consistency over multiple data stores\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ee.g. If Lupita blocks Miss Angela, then posts, Miss Angela can't see it\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTyphon: A single logical \u003cem\u003eentity\u003c/em\u003e has \u003cem\u003edata item\u003c/em\u003e representations in\ndifferent data stores\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAssumes datastores are serializable or offer atomic read/cas on items\u003c/li\u003e\n\u003cli\u003eTransactions which access same entity AND T1 happens-before T2 get a\ncausal dependency edge T1 -\u0026gt; T2\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCerberus: protocol for transactions involving a single entity x\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWrites can only affect one representation of x\u003c/li\u003e\n\u003cli\u003eAny number of reads across representations of x\u003c/li\u003e\n\u003cli\u003eGlobal metadata: a version vector for each entity (GVV)\u003c/li\u003e\n\u003cli\u003eEach representation metadata:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUpdate version vector (UVV): versions known as of last update\u003c/li\u003e\n\u003cli\u003eRead version vector (RVV): versions known as of last read\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eConflicts detected when GVV \u0026lt; UVV/RVV\u003c/li\u003e\n\u003cli\u003eTwo phases:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eReads\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCheck GVV for entity x\u003c/li\u003e\n\u003cli\u003ePerform reads of x on each (asked-for) representation\u003c/li\u003e\n\u003cli\u003eAt each representation: check RVV \u0026lt;= GVV, update RVV\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWrites\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSend write to representation\u003c/li\u003e\n\u003cli\u003eCheck UVV \u0026lt;= GVV \u0026amp; RVV \u0026lt;= GVV\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCommit\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUpdate representation and RVV/UVV ensuring RVV/UVV unchanged\u003c/li\u003e\n\u003cli\u003eNew UVV constructed by incrementing i'th entry of existing UVV\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eGeneral-purpose transactions\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf\" rel=\"nofollow\"\u003eCalvin\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSerializable (or strict-1SR) transactions\u003c/li\u003e\n\u003cli\u003eDeterministic txns enqueued into sharded global log\u003c/li\u003e\n\u003cli\u003eLog ensures txn order\u003c/li\u003e\n\u003cli\u003eApplication at replicas/shards requires no further coordination\u003c/li\u003e\n\u003cli\u003eMinimum latency floor for log windows\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/\" rel=\"nofollow\"\u003eCockroachDB\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSerializable\u003c/li\u003e\n\u003cli\u003eAssumes linearizable stores\u003c/li\u003e\n\u003cli\u003eAssumes semi-sync clocks\u003c/li\u003e\n\u003cli\u003eSimilar to a more tractable Spanner\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eMigrations\u003c/h3\u003e\u003ca id=\"user-content-migrations\" class=\"anchor\" aria-label=\"Permalink: Migrations\" href=\"#migrations\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMigrations are hard.\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThere's no silver bullet\u003c/li\u003e\n\u003cli\u003eBut some techniques can make your life easier\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHard cut\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWrite new system and migration to copy old data to it\u003c/li\u003e\n\u003cli\u003eHave dependent services talk to both--but in practice, only one.\u003c/li\u003e\n\u003cli\u003eTurn off old system\u003c/li\u003e\n\u003cli\u003eCopy data\u003c/li\u003e\n\u003cli\u003eStart up new system\u003c/li\u003e\n\u003cli\u003eTradeoffs!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDon't have to worry about in-flight data\u003c/li\u003e\n\u003cli\u003eSimple migration scripts: just read all data and write to new datastore\u003c/li\u003e\n\u003cli\u003eRequires downtime proportional to migration script\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan sometimes scope this to a single shard/user/domain at a time\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIncremental\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWrite new system B\u003c/li\u003e\n\u003cli\u003eDeploy alongside original A\u003c/li\u003e\n\u003cli\u003eDependent services talk to both\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIdeal: find all readers, have every reader talk to both A and B\u003c/li\u003e\n\u003cli\u003eThen start writing to B\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis frees you from having to worry about readers who only know about A\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eConsistency nightmares; need to trace all data dependencies\u003c/li\u003e\n\u003cli\u003eTradeoffs!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eReduced/no downtime\u003c/li\u003e\n\u003cli\u003eBut complex reasoning about data dependencies required\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWrapper services\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOperationally speaking, it can be tricky to FIND and change all users of A\u003c/li\u003e\n\u003cli\u003eSo... don't. Introduce a wrapper service W which proxies to A\u003c/li\u003e\n\u003cli\u003eIntroduce B, and make changes so that W talks to B as well\u003c/li\u003e\n\u003cli\u003eWhen A is phased out, remove W and talk directly to B.\u003c/li\u003e\n\u003cli\u003eAllows for centralized metrics, errors, comparison of behavior, etc\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEventual atomicity\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eImagine you write each update to old service A, then new service B\u003c/li\u003e\n\u003cli\u003eAt some point, your write to A will succeed, and B will fail. What then?\u003c/li\u003e\n\u003cli\u003eCan use read-repair: read A and B, fill in missing updates\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut this requires mergeability: only works for things like CRDTs\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan use a reconciliation process\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIterate over whole DB, look for changes, apply to both.\u003c/li\u003e\n\u003cli\u003eAlso requires something like a CRDT\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan use a Saga\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAll updates go to durable queue\u003c/li\u003e\n\u003cli\u003eQueue worker retries until updates applied to A and B\u003c/li\u003e\n\u003cli\u003eMight require ordering updates to avoid state divergence\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePotentially \u003cem\u003eglobal\u003c/em\u003e serialization\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBe aware of the DB's consistency model\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIsolation\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eImagine Jane writes w1 to A, then B\u003c/li\u003e\n\u003cli\u003eConcurrently, Naomi writes w2 to B, then A\u003c/li\u003e\n\u003cli\u003eResult: A = w2, B = w1\u003c/li\u003e\n\u003cli\u003eEventual atomicity isn't enough to prevent divergence\u003c/li\u003e\n\u003cli\u003eCan reduce issues by picking a standard order: always A then B (or B then A)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut imagine\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJane writes A = w1\u003c/li\u003e\n\u003cli\u003eNaomi writes A = w2\u003c/li\u003e\n\u003cli\u003eNaomi writes B = w2\u003c/li\u003e\n\u003cli\u003eJane writes B = w1\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWe've got a mixed result again. Shoot.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCan mitigate using CRDTs\u003c/li\u003e\n\u003cli\u003eOr, if A and B are sequentially consistent, can use a CaS operation to\nensure agreement on order\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"Write w2 iff the last write was w1\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf operations affect multiple keys, the CaS logic has to be applied at that\nlevel too\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHelpful properties for incremental migrations\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDeterminism\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAvoid having the DB generate random numbers, automatic IDs, timestamps\u003c/li\u003e\n\u003cli\u003eEasier to apply updates to two datastores and get the same results\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIdempotence\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLets you retry updates freely\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCommutativity\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRemoves the need for serializing updates\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eCRDTs: associativity, commutativity, idempotence.\u003c/li\u003e\n\u003cli\u003eImmutability: trivial CRDTs\u003c/li\u003e\n\u003cli\u003eStatelessness: no state to worry about!\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJust make sure you talk to external stateful stuff the same way\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWhat about swapping out queues?\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAs we've touched, queue systems should already be designed for idempotency,\nand ideally commutativity\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf so, this is (relatively) easy\u003c/li\u003e\n\u003cli\u003eWorkers consume from both queues\u003c/li\u003e\n\u003cli\u003eFlip producers to send messages only to the new queue\u003c/li\u003e\n\u003cli\u003eWait for old queue to be exhausted\u003c/li\u003e\n\u003cli\u003eDecommission old queue\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eBut we WANTED order???\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYou're going to need to reconstruct it\u003c/li\u003e\n\u003cli\u003eOne option: single producer tightly coupled to queue\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWrites each message m to both A and B; doesn't move on until both ack\u003c/li\u003e\n\u003cli\u003eThis enforces that both A and B agree on order\u003c/li\u003e\n\u003cli\u003eConsumers can treat A and B identically: consume just from A or B\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAgain, assumes idempotence!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAnother option: sequence numbers, order reconstructed at client\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ee.g. assign sequence numbers to each value\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUse the queue offsets from A?\u003c/li\u003e\n\u003cli\u003eUse a consensus system?\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eClients read sequence numbers, store in internal buffer, apply in order\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h3\u003e\u003ca id=\"user-content-review-5\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-5\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWhen possible, try to use a single node instead of a distributed system. Accept\nthat some failures are unavoidable: SLAs and apologies can be cost-effective.\nTo handle catastrophic failure, we use backups. To improve reliability, we\nintroduce redundancy. To scale to large problems, we divide the problem into\nshards. Immutable values are easy to store and cache, and can be referenced by\nmutable identities, allowing us to build strongly consistent systems at large\nscale. As software grows, different components must scale independently,\nand we break out libraries into distinct services. Service structure goes\nhand-in-hand with teams.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eProduction Concerns\u003c/h2\u003e\u003ca id=\"user-content-production-concerns\" class=\"anchor\" aria-label=\"Permalink: Production Concerns\" href=\"#production-concerns\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMore than design considerations\u003c/li\u003e\n\u003cli\u003eProofs are important, but real systems do IO\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDistributed systems are supported by your culture\u003c/h3\u003e\u003ca id=\"user-content-distributed-systems-are-supported-by-your-culture\" class=\"anchor\" aria-label=\"Permalink: Distributed systems are supported by your culture\" href=\"#distributed-systems-are-supported-by-your-culture\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUnderstanding a distributed system in production requires close cooperation\nof people with many roles\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDevelopment\u003c/li\u003e\n\u003cli\u003eQA\u003c/li\u003e\n\u003cli\u003eOperations\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eEmpathy matters\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDevelopers have to care about production\u003c/li\u003e\n\u003cli\u003eOps has to care about implementation\u003c/li\u003e\n\u003cli\u003eGood communication enables faster diagnosis\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTest everything\u003c/h3\u003e\u003ca id=\"user-content-test-everything\" class=\"anchor\" aria-label=\"Permalink: Test everything\" href=\"#test-everything\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eType systems are great for preventing logical errors\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhich reduces your testing burden\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHowever, they are \u003cem\u003enot\u003c/em\u003e great at predicting or controlling runtime\nperformance\u003c/li\u003e\n\u003cli\u003eSo, you need a solid test suite\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIdeally, you want a \u003cem\u003eslider\u003c/em\u003e for rigorousness\u003c/li\u003e\n\u003cli\u003eQuick example-based tests that run in a few seconds\u003c/li\u003e\n\u003cli\u003eMore thorough property-based tests that can run overnight\u003c/li\u003e\n\u003cli\u003eBe able to simulate an entire cluster in-process\u003c/li\u003e\n\u003cli\u003eControl concurrent interleavings with simulated networks\u003c/li\u003e\n\u003cli\u003eAutomated hardware faults\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTesting distributed systems is much, much harder than testing local ones\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHuge swath of failure modes you've never even heard of\u003c/li\u003e\n\u003cli\u003eCombinatorial state spaces\u003c/li\u003e\n\u003cli\u003eBugs can manifest only for small/large/intermediate time/space/concurrency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003e\"It's Slow\"\u003c/h3\u003e\u003ca id=\"user-content-its-slow\" class=\"anchor\" aria-label=\"Permalink: \u0026quot;It's Slow\u0026quot;\" href=\"#its-slow\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJeff Hodges: The worst bug you'll ever hear is \"it's slow\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHappens all the time, really difficult to localize\u003c/li\u003e\n\u003cli\u003eBecause the system is distributed, have to profile multiple nodes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNot many profilers are built for this\u003c/li\u003e\n\u003cli\u003eSigelman et al, 2010: Dapper, a Large-Scale Distributed Systems Tracing\nInfrastructure\u003c/li\u003e\n\u003cli\u003eZipkin\u003c/li\u003e\n\u003cli\u003eBig tooling investment\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eProfilers are good at finding CPU problems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut high latency is often a sign of IO, not CPU\u003c/li\u003e\n\u003cli\u003eDisk latency\u003c/li\u003e\n\u003cli\u003eNetwork latency\u003c/li\u003e\n\u003cli\u003eGC latency\u003c/li\u003e\n\u003cli\u003eQueue latency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTry to localize problem using application-level metrics\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThen dig in to process and OS performance\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eLatency variance between nodes doing the same work is an important signal\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e1/3 nodes slow: likely node HW, re-route\u003c/li\u003e\n\u003cli\u003e3/3 nodes slow: likely a logical fault: look at shard size, workload, queries\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eTail latencies are magnified by fanout workloads\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://research.google.com/pubs/pub40801.html\" rel=\"nofollow\"\u003eJeff Dean, 2013: The Tail at Scale\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eConsider speculative parallelism\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eInstrument everything\u003c/h3\u003e\u003ca id=\"user-content-instrument-everything\" class=\"anchor\" aria-label=\"Permalink: Instrument everything\" href=\"#instrument-everything\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSlowness (and outright errors) in prod stem from the interactions \u003cem\u003ebetween\u003c/em\u003e\nsystems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhy? Because your thorough test suite probably verified that the single\nsystem was mostly correct\u003c/li\u003e\n\u003cli\u003eSo we need a way to understand what the system is doing in prod\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIn relation to its dependencies\u003c/li\u003e\n\u003cli\u003eWhich can, in turn, drive new tests\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIn a way, good monitoring is like continuous testing\u003c/li\u003e\n\u003cli\u003eBut not a replacement: these are distinct domains\u003c/li\u003e\n\u003cli\u003eBoth provide assurance that your changes are OK\u003c/li\u003e\n\u003cli\u003eWant high-frequency monitoring\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProduction behaviors can take place on 1ms scales\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTCP incast\u003c/li\u003e\n\u003cli\u003e~1ms resolution \u003cem\u003eideally\u003c/em\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOps response time, in the limit, scales linearly with observation latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e~1 second end to end latency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIdeally, millisecond latencies, maybe ms resolution too\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eUsually cost-prohibitive; back off to 1s or 10s\u003c/li\u003e\n\u003cli\u003eSometimes you can tolerate 60s\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAnd for capacity planning, hourly/daily seasonality is more useful\u003c/li\u003e\n\u003cli\u003eInstrumentation should be tightly coupled to the app\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMeasure only what matters\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eResponding to requests is important\u003c/li\u003e\n\u003cli\u003eNode CPU doesn't matter as much\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eKey metrics for most systems\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eApdex: successful response WITHIN latency SLA\u003c/li\u003e\n\u003cli\u003eLatency profiles: 0, 0.5, 0.95, 0.99, 1\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePercentiles, not means\u003c/li\u003e\n\u003cli\u003eBTW you can't take the mean of percentiles either\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOverall throughput\u003c/li\u003e\n\u003cli\u003eQueue statistics\u003c/li\u003e\n\u003cli\u003eQueries per query\u003c/li\u003e\n\u003cli\u003eSubjective experience of other systems latency/throughput\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe DB might think it's healthy, but clients could see it as slow\u003c/li\u003e\n\u003cli\u003eCombinatorial explosion--best to use this when drilling into a failure\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eYou probably have to write this instrumentation yourself\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eInvest in a metrics library\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eOut-of-the-box monitoring usually doesn't measure what really matters: your\napp's behavior\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut it can be really useful in tracking down causes of problems\u003c/li\u003e\n\u003cli\u003eHost metrics like CPU, disk, etc\u003c/li\u003e\n\u003cli\u003eWhere your app does something common (e.g. rails apps) tools like New\nRelic work well\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eShard metrics by client\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHelpful when users have varying workloads\u003c/li\u003e\n\u003cli\u003eCan tune thresholds to be appropriate for that client\u003c/li\u003e\n\u003cli\u003eA few for major clients, another bucket for \"the rest\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSuperpower: distributed tracing infra (Zipkin, Dapper, etc)\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSignificant time investment\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-chow.pdf\" rel=\"nofollow\"\u003eMystery Machine\u003c/a\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAutomatic inference of causal relationships between services from trace data\u003c/li\u003e\n\u003cli\u003eIdentification of critical paths\u003c/li\u003e\n\u003cli\u003ePerformance modeling new algorithms before implementation\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eLogging\u003c/h3\u003e\u003ca id=\"user-content-logging\" class=\"anchor\" aria-label=\"Permalink: Logging\" href=\"#logging\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLogging is less useful at scale\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProblems may not be localized to one node\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAs requests touch more services, must trace through many logfiles\u003c/li\u003e\n\u003cli\u003eInvest in log collection infrastructure\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eELK, Splunk, etc\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUnstructured information is harder to aggregate\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLog structured events\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eShadow traffic\u003c/h3\u003e\u003ca id=\"user-content-shadow-traffic\" class=\"anchor\" aria-label=\"Permalink: Shadow traffic\" href=\"#shadow-traffic\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLoad tests are only useful insofar as the simulated load matches the actual\nload\u003c/li\u003e\n\u003cli\u003eConsider dumping production traffic\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAwesome: kill a process with SIGUSR1, it dumps five minutes of request load\u003c/li\u003e\n\u003cli\u003eAwesome: tcpdump/tcpreplay harnesses for requests\u003c/li\u003e\n\u003cli\u003eAwesome: shadowing live prod traffic to your staging/QA nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eSee Envoy from Lyft\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eVersioning\u003c/h3\u003e\u003ca id=\"user-content-versioning\" class=\"anchor\" aria-label=\"Permalink: Versioning\" href=\"#versioning\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eProtocol versioning is, as far as I know, a wide-open problem\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDo include a version tag with all messages\u003c/li\u003e\n\u003cli\u003eDo include compatibility logic\u003c/li\u003e\n\u003cli\u003eInform clients when their request can't be honored\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAnd instrument this so you know which systems have to be upgraded\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eRollouts\u003c/h3\u003e\u003ca id=\"user-content-rollouts\" class=\"anchor\" aria-label=\"Permalink: Rollouts\" href=\"#rollouts\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRollouts are often how you fix problems\u003c/li\u003e\n\u003cli\u003eSpend the time to get automated, reliable deploys\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAmplifies everything else you do\u003c/li\u003e\n\u003cli\u003eHave nodes smoothly cycle through to prevent traffic interruption\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis implies you'll have multiple versions of your software running at\nonce\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eVersioning rears its ugly head\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eInform load balancer that they're going out of rotation\u003c/li\u003e\n\u003cli\u003eCoordinate to prevent cascading failures\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRoll out only to a fraction of load or fraction of users\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGradually ramp up number of users on the new software\u003c/li\u003e\n\u003cli\u003eEither revert or roll forward when you see errors\u003c/li\u003e\n\u003cli\u003eConsider shadowing traffic in prod and comparing old/new versions\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGood way to determine if new code is faster \u0026amp; correct\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAutomated Control\u003c/h3\u003e\u003ca id=\"user-content-automated-control\" class=\"anchor\" aria-label=\"Permalink: Automated Control\" href=\"#automated-control\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAutomated failure handling is good\u003c/li\u003e\n\u003cli\u003eBut not too much\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"Ironies of Automation\", Bainbridge 1983\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://pdfs.semanticscholar.org/0713/bb9d9b138e4e0a15406006de9b0cddf68e28.pdf\" rel=\"nofollow\"\u003ehttps://pdfs.semanticscholar.org/0713/bb9d9b138e4e0a15406006de9b0cddf68e28.pdf\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eControlling complex systems requires operators with accurate mental models\nof how the system is behaving, ought to behave, and how it responds to\ncontrol inputs\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIf you aren't engaged in the control process, you forget these things\u003c/li\u003e\n\u003cli\u003eHuman takeover is then challenging!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eHumans literally cannot pay attention to things that don't change for more\nthan ~30 minutes\u003c/li\u003e\n\u003cli\u003eMerely monitoring, as opposed to running, a complex process \"deskills\"\noperators\u003c/li\u003e\n\u003cli\u003eSo... consider \u003cem\u003edeliberately\u003c/em\u003e bringing humans into the control process\u003c/li\u003e\n\u003cli\u003eCatastrophic failure is easy to identify\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut automated control tends to mask failure: prefailure trends are not\napparent until well outside the automated-control regime\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eIf humans are expected to verify automated control decisions, humans must\nbe \u003cem\u003ecapable\u003c/em\u003e of verifying that reasoning\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHuge ML models (for example) make inscrutable decisions\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eFeature flags\u003c/h3\u003e\u003ca id=\"user-content-feature-flags\" class=\"anchor\" aria-label=\"Permalink: Feature flags\" href=\"#feature-flags\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWe want incremental rollouts of a changeset after a deploy\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIntroduce features one by one to watch their impact on metrics\u003c/li\u003e\n\u003cli\u003eGradually shift load from one database to another\u003c/li\u003e\n\u003cli\u003eDisable features when rollout goes wrong\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWe want to obtain partial availability when some services are degraded\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDisable expensive features to speed recovery during a failure\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eUse a highly available coordination service to decide which codepaths to\nenable, or how often to take them\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis service should have minimal dependencies\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDon't use the primary DB\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eWhen things go wrong, you can \u003cem\u003etune\u003c/em\u003e the system's behavior\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhen coordination service is down, fail \u003cem\u003esafe\u003c/em\u003e!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eChaos engineering\u003c/h3\u003e\u003ca id=\"user-content-chaos-engineering\" class=\"anchor\" aria-label=\"Permalink: Chaos engineering\" href=\"#chaos-engineering\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBreaking things in production\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eForces engineers to handle failure appropriately \u003cem\u003enow\u003c/em\u003e, not in response to\nan incident later\u003c/li\u003e\n\u003cli\u003eIdentifies unexpected dependencies in the critical path\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\"When the new stats service goes down, it takes the API with it. Are you\n\u003cem\u003esure\u003c/em\u003e that's necessary?\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRequires good instrumentation and alerting, so you can measure impact of\nevents\u003c/li\u003e\n\u003cli\u003eLimited blast radius\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDon't nuke an entire datacenter every five minutes\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBut \u003cem\u003edo\u003c/em\u003e try it once a quarter\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eDon't break \u003cem\u003etoo\u003c/em\u003e many nodes in a replication group\u003c/li\u003e\n\u003cli\u003eBreak only a small fraction of requests/users at a time\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eOh no, queues\u003c/h3\u003e\u003ca id=\"user-content-oh-no-queues\" class=\"anchor\" aria-label=\"Permalink: Oh no, queues\" href=\"#oh-no-queues\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEvery queue is a place for things to go horribly, horribly wrong\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNo node has unbounded memory. Your queues \u003cem\u003emust\u003c/em\u003e be bounded\u003c/li\u003e\n\u003cli\u003eBut how big? Nobody knows\u003c/li\u003e\n\u003cli\u003eInstrument your queues in prod to find out\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eLittle's Law: mean queue depth = mean arrival rate * mean latency\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThis is distribution-independent!\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eQueues exist to smooth out fluctuations in load\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eImproves throughput at expense of latency\u003c/li\u003e\n\u003cli\u003eIf your load is higher than capacity, no queue will save you\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"https://ferd.ca/queues-don-t-fix-overload.html\" rel=\"nofollow\"\u003ehttps://ferd.ca/queues-don-t-fix-overload.html\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eShed load or apply backpressure when queues become full\u003c/li\u003e\n\u003cli\u003eInstrument this\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWhen load-shedding occurs, alarm bells should ring\u003c/li\u003e\n\u003cli\u003eBackpressure is visible as upstream latency\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eInstrument queue depths\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHigh depths is a clue that you need to add node capacity\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEnd to end queue latency should be smaller than fluctuation timescales\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eRaising the queue size can be tempting, but is a vicious cycle\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003eAll of this is HARD. I don't have good answers for you\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAsk Jeff Hodges why it's hard: see his RICON West 2013 talk\u003c/li\u003e\n\u003cli\u003eSee Zach Tellman - Everything Will Flow\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eReview\u003c/h2\u003e\u003ca id=\"user-content-review-6\" class=\"anchor\" aria-label=\"Permalink: Review\" href=\"#review-6\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eRunning distributed systems requires cooperation between developers, QA, and\noperations engineers. Static analysis, and a test suite including example- and\nproperty-based tests, can help ensure program correctness, but understanding\nproduction behavior requires comprehensive instrumentation and alerting. Mature\ndistributed systems teams often invest in tooling: traffic shadowing,\nversioning, incremental deploys, and feature flags. Finally, queues require\nspecial care.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eFurther reading\u003c/h2\u003e\u003ca id=\"user-content-further-reading\" class=\"anchor\" aria-label=\"Permalink: Further reading\" href=\"#further-reading\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eOnline\u003c/h3\u003e\u003ca id=\"user-content-online\" class=\"anchor\" aria-label=\"Permalink: Online\" href=\"#online\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMixu has a delightful book on distributed systems with incredible detail. \u003ca href=\"http://book.mixu.net/distsys/\" rel=\"nofollow\"\u003ehttp://book.mixu.net/distsys/\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eJeff Hodges has some excellent, production-focused advice. \u003ca href=\"https://www.somethingsimilar.com/2013/01/14/notes-on-distributed-systems-for-young-bloods/\" rel=\"nofollow\"\u003ehttps://www.somethingsimilar.com/2013/01/14/notes-on-distributed-systems-for-young-bloods/\u003c/a\u003e \u003ca href=\"https://player.vimeo.com/video/42898664\" rel=\"nofollow\"\u003ehttps://player.vimeo.com/video/42898664\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eThe Fallacies of Distributed Computing is a classic text on mistaken assumptions we make designing distributed systems. \u003ca href=\"http://www.rgoarchitects.com/Files/fallacies.pdf\" rel=\"nofollow\"\u003ehttp://www.rgoarchitects.com/Files/fallacies.pdf\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eChristopher Meiklejohn has a list of key papers in distributed systems. \u003ca href=\"http://christophermeiklejohn.com/distributed/systems/2013/07/12/readings-in-distributed-systems.html\" rel=\"nofollow\"\u003ehttp://christophermeiklejohn.com/distributed/systems/2013/07/12/readings-in-distributed-systems.html\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eDan Creswell has a lovely reading list. \u003ca href=\"https://dancres.github.io/Pages/\" rel=\"nofollow\"\u003ehttps://dancres.github.io/Pages/\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTrees\u003c/h3\u003e\u003ca id=\"user-content-trees\" class=\"anchor\" aria-label=\"Permalink: Trees\" href=\"#trees\"\u003e\u003csvg data-component=\"Octicon\" class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMartin Kleppmann's \u003ca href=\"https://dataintensive.net/\" rel=\"nofollow\"\u003eDesigning Data-Intensive\nApplications\u003c/a\u003e offers a thorough tour of\ndistributed systems for practitioners.\u003c/li\u003e\n\u003cli\u003eNancy Lynch's \"Distributed Algorithms\" is a comprehensive overview of the\nfield from a more theoretical perspective\u003c/li\u003e\n\u003cli\u003eSuggestions from students:\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDonella Meadows' \"Thinking in Systems\"\u003c/li\u003e\n\u003cli\u003e\"Database Internals: A Deep Dive into How Distributed Data Systems Work\"\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePerhaps more intense than Kleppman\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\"Thinking in Systems\"\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/article\u003e","richTextTruncated":false,"renderedFileInfo":null,"symbols":{"timed_out":false,"not_analyzed":false,"symbols":[{"name":"An Introduction to Distributed Systems","fully_qualified_name":"An Introduction to Distributed Systems","kind":"section_1","ident_start":2,"ident_end":40,"extent_start":0,"extent_end":80386,"ident_utf16":{"start":{"line_number":0,"utf16_col":2},"end":{"line_number":0,"utf16_col":40}},"extent_utf16":{"start":{"line_number":0,"utf16_col":0},"end":{"line_number":1894,"utf16_col":0}}},{"name":"Intro","fully_qualified_name":"Intro","kind":"section_2","ident_start":566,"ident_end":571,"extent_start":563,"extent_end":3256,"ident_utf16":{"start":{"line_number":12,"utf16_col":3},"end":{"line_number":12,"utf16_col":8}},"extent_utf16":{"start":{"line_number":12,"utf16_col":0},"end":{"line_number":74,"utf16_col":0}}},{"name":"What makes a thing distributed?","fully_qualified_name":"What makes a thing distributed?","kind":"section_2","ident_start":3259,"ident_end":3290,"extent_start":3256,"extent_end":4247,"ident_utf16":{"start":{"line_number":74,"utf16_col":3},"end":{"line_number":74,"utf16_col":34}},"extent_utf16":{"start":{"line_number":74,"utf16_col":0},"end":{"line_number":106,"utf16_col":0}}},{"name":"Nodes and networks","fully_qualified_name":"Nodes and networks","kind":"section_2","ident_start":4250,"ident_end":4268,"extent_start":4247,"extent_end":7233,"ident_utf16":{"start":{"line_number":106,"utf16_col":3},"end":{"line_number":106,"utf16_col":21}},"extent_utf16":{"start":{"line_number":106,"utf16_col":0},"end":{"line_number":187,"utf16_col":0}}},{"name":"Nodes","fully_qualified_name":"Nodes","kind":"section_3","ident_start":4381,"ident_end":4386,"extent_start":4377,"extent_end":5404,"ident_utf16":{"start":{"line_number":111,"utf16_col":4},"end":{"line_number":111,"utf16_col":9}},"extent_utf16":{"start":{"line_number":111,"utf16_col":0},"end":{"line_number":137,"utf16_col":0}}},{"name":"Networks as message flows","fully_qualified_name":"Networks as message flows","kind":"section_3","ident_start":5408,"ident_end":5433,"extent_start":5404,"extent_end":5970,"ident_utf16":{"start":{"line_number":137,"utf16_col":4},"end":{"line_number":137,"utf16_col":29}},"extent_utf16":{"start":{"line_number":137,"utf16_col":0},"end":{"line_number":152,"utf16_col":0}}},{"name":"Causality diagrams","fully_qualified_name":"Causality diagrams","kind":"section_3","ident_start":5974,"ident_end":5992,"extent_start":5970,"extent_end":6238,"ident_utf16":{"start":{"line_number":152,"utf16_col":4},"end":{"line_number":152,"utf16_col":22}},"extent_utf16":{"start":{"line_number":152,"utf16_col":0},"end":{"line_number":159,"utf16_col":0}}},{"name":"Synchronous networks","fully_qualified_name":"Synchronous networks","kind":"section_3","ident_start":6242,"ident_end":6262,"extent_start":6238,"extent_end":6455,"ident_utf16":{"start":{"line_number":159,"utf16_col":4},"end":{"line_number":159,"utf16_col":24}},"extent_utf16":{"start":{"line_number":159,"utf16_col":0},"end":{"line_number":167,"utf16_col":0}}},{"name":"Semi-synchronous networks","fully_qualified_name":"Semi-synchronous networks","kind":"section_3","ident_start":6459,"ident_end":6484,"extent_start":6455,"extent_end":6557,"ident_utf16":{"start":{"line_number":167,"utf16_col":4},"end":{"line_number":167,"utf16_col":29}},"extent_utf16":{"start":{"line_number":167,"utf16_col":0},"end":{"line_number":171,"utf16_col":0}}},{"name":"Asynchronous networks","fully_qualified_name":"Asynchronous networks","kind":"section_3","ident_start":6561,"ident_end":6582,"extent_start":6557,"extent_end":7233,"ident_utf16":{"start":{"line_number":171,"utf16_col":4},"end":{"line_number":171,"utf16_col":25}},"extent_utf16":{"start":{"line_number":171,"utf16_col":0},"end":{"line_number":187,"utf16_col":0}}},{"name":"When networks go wrong","fully_qualified_name":"When networks go wrong","kind":"section_2","ident_start":7236,"ident_end":7258,"extent_start":7233,"extent_end":7765,"ident_utf16":{"start":{"line_number":187,"utf16_col":3},"end":{"line_number":187,"utf16_col":25}},"extent_utf16":{"start":{"line_number":187,"utf16_col":0},"end":{"line_number":202,"utf16_col":0}}},{"name":"Low level protocols","fully_qualified_name":"Low level protocols","kind":"section_2","ident_start":7768,"ident_end":7787,"extent_start":7765,"extent_end":9237,"ident_utf16":{"start":{"line_number":202,"utf16_col":3},"end":{"line_number":202,"utf16_col":22}},"extent_utf16":{"start":{"line_number":202,"utf16_col":0},"end":{"line_number":240,"utf16_col":0}}},{"name":"TCP","fully_qualified_name":"TCP","kind":"section_3","ident_start":7793,"ident_end":7796,"extent_start":7789,"extent_end":8280,"ident_utf16":{"start":{"line_number":204,"utf16_col":4},"end":{"line_number":204,"utf16_col":7}},"extent_utf16":{"start":{"line_number":204,"utf16_col":0},"end":{"line_number":217,"utf16_col":0}}},{"name":"UDP","fully_qualified_name":"UDP","kind":"section_3","ident_start":8284,"ident_end":8287,"extent_start":8280,"extent_end":9237,"ident_utf16":{"start":{"line_number":217,"utf16_col":4},"end":{"line_number":217,"utf16_col":7}},"extent_utf16":{"start":{"line_number":217,"utf16_col":0},"end":{"line_number":240,"utf16_col":0}}},{"name":"Clocks","fully_qualified_name":"Clocks","kind":"section_2","ident_start":9240,"ident_end":9246,"extent_start":9237,"extent_end":13272,"ident_utf16":{"start":{"line_number":240,"utf16_col":3},"end":{"line_number":240,"utf16_col":9}},"extent_utf16":{"start":{"line_number":240,"utf16_col":0},"end":{"line_number":330,"utf16_col":0}}},{"name":"Wall Clocks","fully_qualified_name":"Wall Clocks","kind":"section_3","ident_start":9403,"ident_end":9414,"extent_start":9399,"extent_end":11161,"ident_utf16":{"start":{"line_number":246,"utf16_col":4},"end":{"line_number":246,"utf16_col":15}},"extent_utf16":{"start":{"line_number":246,"utf16_col":0},"end":{"line_number":278,"utf16_col":0}}},{"name":"Lamport Clocks","fully_qualified_name":"Lamport Clocks","kind":"section_3","ident_start":11165,"ident_end":11179,"extent_start":11161,"extent_end":11558,"ident_utf16":{"start":{"line_number":278,"utf16_col":4},"end":{"line_number":278,"utf16_col":18}},"extent_utf16":{"start":{"line_number":278,"utf16_col":0},"end":{"line_number":289,"utf16_col":0}}},{"name":"Vector Clocks","fully_qualified_name":"Vector Clocks","kind":"section_3","ident_start":11562,"ident_end":11575,"extent_start":11558,"extent_end":12479,"ident_utf16":{"start":{"line_number":289,"utf16_col":4},"end":{"line_number":289,"utf16_col":17}},"extent_utf16":{"start":{"line_number":289,"utf16_col":0},"end":{"line_number":311,"utf16_col":0}}},{"name":"GPS \u0026 Atomic Clocks","fully_qualified_name":"GPS \u0026 Atomic Clocks","kind":"section_3","ident_start":12483,"ident_end":12502,"extent_start":12479,"extent_end":13272,"ident_utf16":{"start":{"line_number":311,"utf16_col":4},"end":{"line_number":311,"utf16_col":23}},"extent_utf16":{"start":{"line_number":311,"utf16_col":0},"end":{"line_number":330,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_2","ident_start":13275,"ident_end":13281,"extent_start":13272,"extent_end":13643,"ident_utf16":{"start":{"line_number":330,"utf16_col":3},"end":{"line_number":330,"utf16_col":9}},"extent_utf16":{"start":{"line_number":330,"utf16_col":0},"end":{"line_number":341,"utf16_col":0}}},{"name":"Availability","fully_qualified_name":"Availability","kind":"section_2","ident_start":13646,"ident_end":13658,"extent_start":13643,"extent_end":15074,"ident_utf16":{"start":{"line_number":341,"utf16_col":3},"end":{"line_number":341,"utf16_col":15}},"extent_utf16":{"start":{"line_number":341,"utf16_col":0},"end":{"line_number":387,"utf16_col":0}}},{"name":"Total availability","fully_qualified_name":"Total availability","kind":"section_3","ident_start":13745,"ident_end":13763,"extent_start":13741,"extent_end":13916,"ident_utf16":{"start":{"line_number":345,"utf16_col":4},"end":{"line_number":345,"utf16_col":22}},"extent_utf16":{"start":{"line_number":345,"utf16_col":0},"end":{"line_number":351,"utf16_col":0}}},{"name":"Sticky availability","fully_qualified_name":"Sticky availability","kind":"section_3","ident_start":13920,"ident_end":13939,"extent_start":13916,"extent_end":14063,"ident_utf16":{"start":{"line_number":351,"utf16_col":4},"end":{"line_number":351,"utf16_col":23}},"extent_utf16":{"start":{"line_number":351,"utf16_col":0},"end":{"line_number":356,"utf16_col":0}}},{"name":"High availability","fully_qualified_name":"High availability","kind":"section_3","ident_start":14067,"ident_end":14084,"extent_start":14063,"extent_end":14216,"ident_utf16":{"start":{"line_number":356,"utf16_col":4},"end":{"line_number":356,"utf16_col":21}},"extent_utf16":{"start":{"line_number":356,"utf16_col":0},"end":{"line_number":362,"utf16_col":0}}},{"name":"Majority available","fully_qualified_name":"Majority available","kind":"section_3","ident_start":14220,"ident_end":14238,"extent_start":14216,"extent_end":14393,"ident_utf16":{"start":{"line_number":362,"utf16_col":4},"end":{"line_number":362,"utf16_col":22}},"extent_utf16":{"start":{"line_number":362,"utf16_col":0},"end":{"line_number":368,"utf16_col":0}}},{"name":"Quantifying availability","fully_qualified_name":"Quantifying availability","kind":"section_3","ident_start":14397,"ident_end":14421,"extent_start":14393,"extent_end":15074,"ident_utf16":{"start":{"line_number":368,"utf16_col":4},"end":{"line_number":368,"utf16_col":28}},"extent_utf16":{"start":{"line_number":368,"utf16_col":0},"end":{"line_number":387,"utf16_col":0}}},{"name":"Consistency","fully_qualified_name":"Consistency","kind":"section_2","ident_start":15077,"ident_end":15088,"extent_start":15074,"extent_end":21325,"ident_utf16":{"start":{"line_number":387,"utf16_col":3},"end":{"line_number":387,"utf16_col":14}},"extent_utf16":{"start":{"line_number":387,"utf16_col":0},"end":{"line_number":537,"utf16_col":0}}},{"name":"Monotonic Reads","fully_qualified_name":"Monotonic Reads","kind":"section_3","ident_start":15172,"ident_end":15187,"extent_start":15168,"extent_end":15272,"ident_utf16":{"start":{"line_number":391,"utf16_col":4},"end":{"line_number":391,"utf16_col":19}},"extent_utf16":{"start":{"line_number":391,"utf16_col":0},"end":{"line_number":395,"utf16_col":0}}},{"name":"Monotonic Writes","fully_qualified_name":"Monotonic Writes","kind":"section_3","ident_start":15276,"ident_end":15292,"extent_start":15272,"extent_end":15387,"ident_utf16":{"start":{"line_number":395,"utf16_col":4},"end":{"line_number":395,"utf16_col":20}},"extent_utf16":{"start":{"line_number":395,"utf16_col":0},"end":{"line_number":400,"utf16_col":0}}},{"name":"Read Your Writes","fully_qualified_name":"Read Your Writes","kind":"section_3","ident_start":15391,"ident_end":15407,"extent_start":15387,"extent_end":15507,"ident_utf16":{"start":{"line_number":400,"utf16_col":4},"end":{"line_number":400,"utf16_col":20}},"extent_utf16":{"start":{"line_number":400,"utf16_col":0},"end":{"line_number":405,"utf16_col":0}}},{"name":"Writes Follow Reads","fully_qualified_name":"Writes Follow Reads","kind":"section_3","ident_start":15511,"ident_end":15530,"extent_start":15507,"extent_end":15609,"ident_utf16":{"start":{"line_number":405,"utf16_col":4},"end":{"line_number":405,"utf16_col":23}},"extent_utf16":{"start":{"line_number":405,"utf16_col":0},"end":{"line_number":409,"utf16_col":0}}},{"name":"Causal consistency","fully_qualified_name":"Causal consistency","kind":"section_3","ident_start":15613,"ident_end":15631,"extent_start":15609,"extent_end":16039,"ident_utf16":{"start":{"line_number":409,"utf16_col":4},"end":{"line_number":409,"utf16_col":22}},"extent_utf16":{"start":{"line_number":409,"utf16_col":0},"end":{"line_number":419,"utf16_col":0}}},{"name":"Sequential consistency","fully_qualified_name":"Sequential consistency","kind":"section_3","ident_start":16043,"ident_end":16065,"extent_start":16039,"extent_end":16305,"ident_utf16":{"start":{"line_number":419,"utf16_col":4},"end":{"line_number":419,"utf16_col":26}},"extent_utf16":{"start":{"line_number":419,"utf16_col":0},"end":{"line_number":427,"utf16_col":0}}},{"name":"Linearizability","fully_qualified_name":"Linearizability","kind":"section_3","ident_start":16309,"ident_end":16324,"extent_start":16305,"extent_end":16578,"ident_utf16":{"start":{"line_number":427,"utf16_col":4},"end":{"line_number":427,"utf16_col":19}},"extent_utf16":{"start":{"line_number":427,"utf16_col":0},"end":{"line_number":435,"utf16_col":0}}},{"name":"Transactional Models","fully_qualified_name":"Transactional Models","kind":"section_3","ident_start":16582,"ident_end":16602,"extent_start":16578,"extent_end":19960,"ident_utf16":{"start":{"line_number":435,"utf16_col":4},"end":{"line_number":435,"utf16_col":24}},"extent_utf16":{"start":{"line_number":435,"utf16_col":0},"end":{"line_number":511,"utf16_col":0}}},{"name":"Does any of this actually matter?","fully_qualified_name":"Does any of this actually matter?","kind":"section_3","ident_start":19964,"ident_end":19997,"extent_start":19960,"extent_end":21325,"ident_utf16":{"start":{"line_number":511,"utf16_col":4},"end":{"line_number":511,"utf16_col":37}},"extent_utf16":{"start":{"line_number":511,"utf16_col":0},"end":{"line_number":537,"utf16_col":0}}},{"name":"Tradeoffs","fully_qualified_name":"Tradeoffs","kind":"section_2","ident_start":21328,"ident_end":21337,"extent_start":21325,"extent_end":24110,"ident_utf16":{"start":{"line_number":537,"utf16_col":3},"end":{"line_number":537,"utf16_col":12}},"extent_utf16":{"start":{"line_number":537,"utf16_col":0},"end":{"line_number":612,"utf16_col":0}}},{"name":"Availability and Consistency","fully_qualified_name":"Availability and Consistency","kind":"section_3","ident_start":21731,"ident_end":21759,"extent_start":21727,"extent_end":22357,"ident_utf16":{"start":{"line_number":548,"utf16_col":4},"end":{"line_number":548,"utf16_col":32}},"extent_utf16":{"start":{"line_number":548,"utf16_col":0},"end":{"line_number":572,"utf16_col":0}}},{"name":"Harvest and Yield","fully_qualified_name":"Harvest and Yield","kind":"section_3","ident_start":22361,"ident_end":22378,"extent_start":22357,"extent_end":23287,"ident_utf16":{"start":{"line_number":572,"utf16_col":4},"end":{"line_number":572,"utf16_col":21}},"extent_utf16":{"start":{"line_number":572,"utf16_col":0},"end":{"line_number":591,"utf16_col":0}}},{"name":"Hybrid systems","fully_qualified_name":"Hybrid systems","kind":"section_3","ident_start":23291,"ident_end":23305,"extent_start":23287,"extent_end":23778,"ident_utf16":{"start":{"line_number":591,"utf16_col":4},"end":{"line_number":591,"utf16_col":18}},"extent_utf16":{"start":{"line_number":591,"utf16_col":0},"end":{"line_number":603,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_3","ident_start":23782,"ident_end":23788,"extent_start":23778,"extent_end":24110,"ident_utf16":{"start":{"line_number":603,"utf16_col":4},"end":{"line_number":603,"utf16_col":10}},"extent_utf16":{"start":{"line_number":603,"utf16_col":0},"end":{"line_number":612,"utf16_col":0}}},{"name":"Avoid Consensus Wherever Possible","fully_qualified_name":"Avoid Consensus Wherever Possible","kind":"section_2","ident_start":24113,"ident_end":24146,"extent_start":24110,"extent_end":28177,"ident_utf16":{"start":{"line_number":612,"utf16_col":3},"end":{"line_number":612,"utf16_col":36}},"extent_utf16":{"start":{"line_number":612,"utf16_col":0},"end":{"line_number":712,"utf16_col":0}}},{"name":"CALM conjecture","fully_qualified_name":"CALM conjecture","kind":"section_3","ident_start":24152,"ident_end":24167,"extent_start":24148,"extent_end":25460,"ident_utf16":{"start":{"line_number":614,"utf16_col":4},"end":{"line_number":614,"utf16_col":19}},"extent_utf16":{"start":{"line_number":614,"utf16_col":0},"end":{"line_number":641,"utf16_col":0}}},{"name":"Gossip","fully_qualified_name":"Gossip","kind":"section_3","ident_start":25464,"ident_end":25470,"extent_start":25460,"extent_end":26376,"ident_utf16":{"start":{"line_number":641,"utf16_col":4},"end":{"line_number":641,"utf16_col":10}},"extent_utf16":{"start":{"line_number":641,"utf16_col":0},"end":{"line_number":667,"utf16_col":0}}},{"name":"CRDTs","fully_qualified_name":"CRDTs","kind":"section_3","ident_start":26380,"ident_end":26385,"extent_start":26376,"extent_end":27515,"ident_utf16":{"start":{"line_number":667,"utf16_col":4},"end":{"line_number":667,"utf16_col":9}},"extent_utf16":{"start":{"line_number":667,"utf16_col":0},"end":{"line_number":695,"utf16_col":0}}},{"name":"HATs","fully_qualified_name":"HATs","kind":"section_3","ident_start":27519,"ident_end":27523,"extent_start":27515,"extent_end":28177,"ident_utf16":{"start":{"line_number":695,"utf16_col":4},"end":{"line_number":695,"utf16_col":8}},"extent_utf16":{"start":{"line_number":695,"utf16_col":0},"end":{"line_number":712,"utf16_col":0}}},{"name":"Fine, We Need Consensus, What Now?","fully_qualified_name":"Fine, We Need Consensus, What Now?","kind":"section_2","ident_start":28180,"ident_end":28214,"extent_start":28177,"extent_end":33466,"ident_utf16":{"start":{"line_number":712,"utf16_col":3},"end":{"line_number":712,"utf16_col":37}},"extent_utf16":{"start":{"line_number":712,"utf16_col":0},"end":{"line_number":839,"utf16_col":0}}},{"name":"Paxos","fully_qualified_name":"Paxos","kind":"section_3","ident_start":29909,"ident_end":29914,"extent_start":29905,"extent_end":31848,"ident_utf16":{"start":{"line_number":757,"utf16_col":4},"end":{"line_number":757,"utf16_col":9}},"extent_utf16":{"start":{"line_number":757,"utf16_col":0},"end":{"line_number":799,"utf16_col":0}}},{"name":"ZAB","fully_qualified_name":"ZAB","kind":"section_3","ident_start":31852,"ident_end":31855,"extent_start":31848,"extent_end":32331,"ident_utf16":{"start":{"line_number":799,"utf16_col":4},"end":{"line_number":799,"utf16_col":7}},"extent_utf16":{"start":{"line_number":799,"utf16_col":0},"end":{"line_number":811,"utf16_col":0}}},{"name":"Humming Consensus","fully_qualified_name":"Humming Consensus","kind":"section_3","ident_start":32335,"ident_end":32352,"extent_start":32331,"extent_end":32495,"ident_utf16":{"start":{"line_number":811,"utf16_col":4},"end":{"line_number":811,"utf16_col":21}},"extent_utf16":{"start":{"line_number":811,"utf16_col":0},"end":{"line_number":817,"utf16_col":0}}},{"name":"Viewstamped Replication","fully_qualified_name":"Viewstamped Replication","kind":"section_3","ident_start":32499,"ident_end":32522,"extent_start":32495,"extent_end":32836,"ident_utf16":{"start":{"line_number":817,"utf16_col":4},"end":{"line_number":817,"utf16_col":27}},"extent_utf16":{"start":{"line_number":817,"utf16_col":0},"end":{"line_number":825,"utf16_col":0}}},{"name":"Raft","fully_qualified_name":"Raft","kind":"section_3","ident_start":32840,"ident_end":32844,"extent_start":32836,"extent_end":33466,"ident_utf16":{"start":{"line_number":825,"utf16_col":4},"end":{"line_number":825,"utf16_col":8}},"extent_utf16":{"start":{"line_number":825,"utf16_col":0},"end":{"line_number":839,"utf16_col":0}}},{"name":"What About Transactions?","fully_qualified_name":"What About Transactions?","kind":"section_2","ident_start":33469,"ident_end":33493,"extent_start":33466,"extent_end":36165,"ident_utf16":{"start":{"line_number":839,"utf16_col":3},"end":{"line_number":839,"utf16_col":27}},"extent_utf16":{"start":{"line_number":839,"utf16_col":0},"end":{"line_number":888,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_2","ident_start":36168,"ident_end":36174,"extent_start":36165,"extent_end":36588,"ident_utf16":{"start":{"line_number":888,"utf16_col":3},"end":{"line_number":888,"utf16_col":9}},"extent_utf16":{"start":{"line_number":888,"utf16_col":0},"end":{"line_number":899,"utf16_col":0}}},{"name":"Characteristic latencies","fully_qualified_name":"Characteristic latencies","kind":"section_2","ident_start":36591,"ident_end":36615,"extent_start":36588,"extent_end":41354,"ident_utf16":{"start":{"line_number":899,"utf16_col":3},"end":{"line_number":899,"utf16_col":27}},"extent_utf16":{"start":{"line_number":899,"utf16_col":0},"end":{"line_number":1007,"utf16_col":0}}},{"name":"Multicore systems","fully_qualified_name":"Multicore systems","kind":"section_3","ident_start":36955,"ident_end":36972,"extent_start":36951,"extent_end":38401,"ident_utf16":{"start":{"line_number":910,"utf16_col":4},"end":{"line_number":910,"utf16_col":21}},"extent_utf16":{"start":{"line_number":910,"utf16_col":0},"end":{"line_number":938,"utf16_col":0}}},{"name":"Local networks","fully_qualified_name":"Local networks","kind":"section_3","ident_start":38405,"ident_end":38419,"extent_start":38401,"extent_end":39306,"ident_utf16":{"start":{"line_number":938,"utf16_col":4},"end":{"line_number":938,"utf16_col":18}},"extent_utf16":{"start":{"line_number":938,"utf16_col":0},"end":{"line_number":961,"utf16_col":0}}},{"name":"Geographic replication","fully_qualified_name":"Geographic replication","kind":"section_3","ident_start":39310,"ident_end":39332,"extent_start":39306,"extent_end":40844,"ident_utf16":{"start":{"line_number":961,"utf16_col":4},"end":{"line_number":961,"utf16_col":26}},"extent_utf16":{"start":{"line_number":961,"utf16_col":0},"end":{"line_number":996,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_3","ident_start":40848,"ident_end":40854,"extent_start":40844,"extent_end":41354,"ident_utf16":{"start":{"line_number":996,"utf16_col":4},"end":{"line_number":996,"utf16_col":10}},"extent_utf16":{"start":{"line_number":996,"utf16_col":0},"end":{"line_number":1007,"utf16_col":0}}},{"name":"Common distributed systems","fully_qualified_name":"Common distributed systems","kind":"section_2","ident_start":41357,"ident_end":41383,"extent_start":41354,"extent_end":45321,"ident_utf16":{"start":{"line_number":1007,"utf16_col":3},"end":{"line_number":1007,"utf16_col":29}},"extent_utf16":{"start":{"line_number":1007,"utf16_col":0},"end":{"line_number":1109,"utf16_col":0}}},{"name":"Outsourced heaps","fully_qualified_name":"Outsourced heaps","kind":"section_3","ident_start":41389,"ident_end":41405,"extent_start":41385,"extent_end":41668,"ident_utf16":{"start":{"line_number":1009,"utf16_col":4},"end":{"line_number":1009,"utf16_col":20}},"extent_utf16":{"start":{"line_number":1009,"utf16_col":0},"end":{"line_number":1018,"utf16_col":0}}},{"name":"KV stores","fully_qualified_name":"KV stores","kind":"section_3","ident_start":41672,"ident_end":41681,"extent_start":41668,"extent_end":42046,"ident_utf16":{"start":{"line_number":1018,"utf16_col":4},"end":{"line_number":1018,"utf16_col":13}},"extent_utf16":{"start":{"line_number":1018,"utf16_col":0},"end":{"line_number":1030,"utf16_col":0}}},{"name":"SQL databases","fully_qualified_name":"SQL databases","kind":"section_3","ident_start":42050,"ident_end":42063,"extent_start":42046,"extent_end":42528,"ident_utf16":{"start":{"line_number":1030,"utf16_col":4},"end":{"line_number":1030,"utf16_col":17}},"extent_utf16":{"start":{"line_number":1030,"utf16_col":0},"end":{"line_number":1041,"utf16_col":0}}},{"name":"Search","fully_qualified_name":"Search","kind":"section_3","ident_start":42532,"ident_end":42538,"extent_start":42528,"extent_end":42732,"ident_utf16":{"start":{"line_number":1041,"utf16_col":4},"end":{"line_number":1041,"utf16_col":10}},"extent_utf16":{"start":{"line_number":1041,"utf16_col":0},"end":{"line_number":1050,"utf16_col":0}}},{"name":"Coordination services","fully_qualified_name":"Coordination services","kind":"section_3","ident_start":42736,"ident_end":42757,"extent_start":42732,"extent_end":42929,"ident_utf16":{"start":{"line_number":1050,"utf16_col":4},"end":{"line_number":1050,"utf16_col":25}},"extent_utf16":{"start":{"line_number":1050,"utf16_col":0},"end":{"line_number":1057,"utf16_col":0}}},{"name":"Streaming systems","fully_qualified_name":"Streaming systems","kind":"section_3","ident_start":42933,"ident_end":42950,"extent_start":42929,"extent_end":43122,"ident_utf16":{"start":{"line_number":1057,"utf16_col":4},"end":{"line_number":1057,"utf16_col":21}},"extent_utf16":{"start":{"line_number":1057,"utf16_col":0},"end":{"line_number":1066,"utf16_col":0}}},{"name":"Distributed queues","fully_qualified_name":"Distributed queues","kind":"section_3","ident_start":43126,"ident_end":43144,"extent_start":43122,"extent_end":44602,"ident_utf16":{"start":{"line_number":1066,"utf16_col":4},"end":{"line_number":1066,"utf16_col":22}},"extent_utf16":{"start":{"line_number":1066,"utf16_col":0},"end":{"line_number":1095,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_3","ident_start":44606,"ident_end":44612,"extent_start":44602,"extent_end":45321,"ident_utf16":{"start":{"line_number":1095,"utf16_col":4},"end":{"line_number":1095,"utf16_col":10}},"extent_utf16":{"start":{"line_number":1095,"utf16_col":0},"end":{"line_number":1109,"utf16_col":0}}},{"name":"A Pattern Language","fully_qualified_name":"A Pattern Language","kind":"section_2","ident_start":45324,"ident_end":45342,"extent_start":45321,"extent_end":68250,"ident_utf16":{"start":{"line_number":1109,"utf16_col":3},"end":{"line_number":1109,"utf16_col":21}},"extent_utf16":{"start":{"line_number":1109,"utf16_col":0},"end":{"line_number":1622,"utf16_col":0}}},{"name":"Don't distribute","fully_qualified_name":"Don't distribute","kind":"section_3","ident_start":45578,"ident_end":45594,"extent_start":45574,"extent_end":46483,"ident_utf16":{"start":{"line_number":1121,"utf16_col":4},"end":{"line_number":1121,"utf16_col":20}},"extent_utf16":{"start":{"line_number":1121,"utf16_col":0},"end":{"line_number":1139,"utf16_col":0}}},{"name":"Use an existing distributed system","fully_qualified_name":"Use an existing distributed system","kind":"section_3","ident_start":46487,"ident_end":46521,"extent_start":46483,"extent_end":46813,"ident_utf16":{"start":{"line_number":1139,"utf16_col":4},"end":{"line_number":1139,"utf16_col":38}},"extent_utf16":{"start":{"line_number":1139,"utf16_col":0},"end":{"line_number":1147,"utf16_col":0}}},{"name":"Never fail","fully_qualified_name":"Never fail","kind":"section_3","ident_start":46817,"ident_end":46827,"extent_start":46813,"extent_end":47188,"ident_utf16":{"start":{"line_number":1147,"utf16_col":4},"end":{"line_number":1147,"utf16_col":14}},"extent_utf16":{"start":{"line_number":1147,"utf16_col":0},"end":{"line_number":1156,"utf16_col":0}}},{"name":"Accept failure","fully_qualified_name":"Accept failure","kind":"section_3","ident_start":47192,"ident_end":47206,"extent_start":47188,"extent_end":47758,"ident_utf16":{"start":{"line_number":1156,"utf16_col":4},"end":{"line_number":1156,"utf16_col":18}},"extent_utf16":{"start":{"line_number":1156,"utf16_col":0},"end":{"line_number":1171,"utf16_col":0}}},{"name":"Recovery First","fully_qualified_name":"Recovery First","kind":"section_3","ident_start":47762,"ident_end":47776,"extent_start":47758,"extent_end":48236,"ident_utf16":{"start":{"line_number":1171,"utf16_col":4},"end":{"line_number":1171,"utf16_col":18}},"extent_utf16":{"start":{"line_number":1171,"utf16_col":0},"end":{"line_number":1182,"utf16_col":0}}},{"name":"Reconciliation Loops","fully_qualified_name":"Reconciliation Loops","kind":"section_3","ident_start":48240,"ident_end":48260,"extent_start":48236,"extent_end":49166,"ident_utf16":{"start":{"line_number":1182,"utf16_col":4},"end":{"line_number":1182,"utf16_col":24}},"extent_utf16":{"start":{"line_number":1182,"utf16_col":0},"end":{"line_number":1201,"utf16_col":0}}},{"name":"Backups","fully_qualified_name":"Backups","kind":"section_3","ident_start":49170,"ident_end":49177,"extent_start":49166,"extent_end":49692,"ident_utf16":{"start":{"line_number":1201,"utf16_col":4},"end":{"line_number":1201,"utf16_col":11}},"extent_utf16":{"start":{"line_number":1201,"utf16_col":0},"end":{"line_number":1214,"utf16_col":0}}},{"name":"Redundancy","fully_qualified_name":"Redundancy","kind":"section_3","ident_start":49696,"ident_end":49706,"extent_start":49692,"extent_end":51317,"ident_utf16":{"start":{"line_number":1214,"utf16_col":4},"end":{"line_number":1214,"utf16_col":14}},"extent_utf16":{"start":{"line_number":1214,"utf16_col":0},"end":{"line_number":1251,"utf16_col":0}}},{"name":"Sharding","fully_qualified_name":"Sharding","kind":"section_3","ident_start":51321,"ident_end":51329,"extent_start":51317,"extent_end":52051,"ident_utf16":{"start":{"line_number":1251,"utf16_col":4},"end":{"line_number":1251,"utf16_col":12}},"extent_utf16":{"start":{"line_number":1251,"utf16_col":0},"end":{"line_number":1270,"utf16_col":0}}},{"name":"Independent domains","fully_qualified_name":"Independent domains","kind":"section_3","ident_start":52055,"ident_end":52074,"extent_start":52051,"extent_end":52625,"ident_utf16":{"start":{"line_number":1270,"utf16_col":4},"end":{"line_number":1270,"utf16_col":23}},"extent_utf16":{"start":{"line_number":1270,"utf16_col":0},"end":{"line_number":1284,"utf16_col":0}}},{"name":"ID structure","fully_qualified_name":"ID structure","kind":"section_3","ident_start":52629,"ident_end":52641,"extent_start":52625,"extent_end":53061,"ident_utf16":{"start":{"line_number":1284,"utf16_col":4},"end":{"line_number":1284,"utf16_col":16}},"extent_utf16":{"start":{"line_number":1284,"utf16_col":0},"end":{"line_number":1298,"utf16_col":0}}},{"name":"Immutable values","fully_qualified_name":"Immutable values","kind":"section_3","ident_start":53065,"ident_end":53081,"extent_start":53061,"extent_end":53675,"ident_utf16":{"start":{"line_number":1298,"utf16_col":4},"end":{"line_number":1298,"utf16_col":20}},"extent_utf16":{"start":{"line_number":1298,"utf16_col":0},"end":{"line_number":1315,"utf16_col":0}}},{"name":"Mutable identities","fully_qualified_name":"Mutable identities","kind":"section_3","ident_start":53679,"ident_end":53697,"extent_start":53675,"extent_end":54568,"ident_utf16":{"start":{"line_number":1315,"utf16_col":4},"end":{"line_number":1315,"utf16_col":22}},"extent_utf16":{"start":{"line_number":1315,"utf16_col":0},"end":{"line_number":1335,"utf16_col":0}}},{"name":"Confluence","fully_qualified_name":"Confluence","kind":"section_3","ident_start":54572,"ident_end":54582,"extent_start":54568,"extent_end":55394,"ident_utf16":{"start":{"line_number":1335,"utf16_col":4},"end":{"line_number":1335,"utf16_col":14}},"extent_utf16":{"start":{"line_number":1335,"utf16_col":0},"end":{"line_number":1350,"utf16_col":0}}},{"name":"Backpressure","fully_qualified_name":"Backpressure","kind":"section_3","ident_start":55398,"ident_end":55410,"extent_start":55394,"extent_end":56524,"ident_utf16":{"start":{"line_number":1350,"utf16_col":4},"end":{"line_number":1350,"utf16_col":16}},"extent_utf16":{"start":{"line_number":1350,"utf16_col":0},"end":{"line_number":1375,"utf16_col":0}}},{"name":"Services for domain models","fully_qualified_name":"Services for domain models","kind":"section_3","ident_start":56528,"ident_end":56554,"extent_start":56524,"extent_end":58018,"ident_utf16":{"start":{"line_number":1375,"utf16_col":4},"end":{"line_number":1375,"utf16_col":30}},"extent_utf16":{"start":{"line_number":1375,"utf16_col":0},"end":{"line_number":1408,"utf16_col":0}}},{"name":"Structure Follows Social Spaces","fully_qualified_name":"Structure Follows Social Spaces","kind":"section_3","ident_start":58022,"ident_end":58053,"extent_start":58018,"extent_end":60817,"ident_utf16":{"start":{"line_number":1408,"utf16_col":4},"end":{"line_number":1408,"utf16_col":35}},"extent_utf16":{"start":{"line_number":1408,"utf16_col":0},"end":{"line_number":1461,"utf16_col":0}}},{"name":"Cross-service coordination","fully_qualified_name":"Cross-service coordination","kind":"section_3","ident_start":60821,"ident_end":60847,"extent_start":60817,"extent_end":63248,"ident_utf16":{"start":{"line_number":1461,"utf16_col":4},"end":{"line_number":1461,"utf16_col":30}},"extent_utf16":{"start":{"line_number":1461,"utf16_col":0},"end":{"line_number":1509,"utf16_col":0}}},{"name":"Migrations","fully_qualified_name":"Migrations","kind":"section_3","ident_start":63252,"ident_end":63262,"extent_start":63248,"extent_end":67596,"ident_utf16":{"start":{"line_number":1509,"utf16_col":4},"end":{"line_number":1509,"utf16_col":14}},"extent_utf16":{"start":{"line_number":1509,"utf16_col":0},"end":{"line_number":1608,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_3","ident_start":67600,"ident_end":67606,"extent_start":67596,"extent_end":68250,"ident_utf16":{"start":{"line_number":1608,"utf16_col":4},"end":{"line_number":1608,"utf16_col":10}},"extent_utf16":{"start":{"line_number":1608,"utf16_col":0},"end":{"line_number":1622,"utf16_col":0}}},{"name":"Production Concerns","fully_qualified_name":"Production Concerns","kind":"section_2","ident_start":68253,"ident_end":68272,"extent_start":68250,"extent_end":78644,"ident_utf16":{"start":{"line_number":1622,"utf16_col":3},"end":{"line_number":1622,"utf16_col":22}},"extent_utf16":{"start":{"line_number":1622,"utf16_col":0},"end":{"line_number":1862,"utf16_col":0}}},{"name":"Distributed systems are supported by your culture","fully_qualified_name":"Distributed systems are supported by your culture","kind":"section_3","ident_start":68360,"ident_end":68409,"extent_start":68356,"extent_end":68708,"ident_utf16":{"start":{"line_number":1627,"utf16_col":4},"end":{"line_number":1627,"utf16_col":53}},"extent_utf16":{"start":{"line_number":1627,"utf16_col":0},"end":{"line_number":1639,"utf16_col":0}}},{"name":"Test everything","fully_qualified_name":"Test everything","kind":"section_3","ident_start":68712,"ident_end":68727,"extent_start":68708,"extent_end":69496,"ident_utf16":{"start":{"line_number":1639,"utf16_col":4},"end":{"line_number":1639,"utf16_col":19}},"extent_utf16":{"start":{"line_number":1639,"utf16_col":0},"end":{"line_number":1657,"utf16_col":0}}},{"name":"\"It's Slow\"","fully_qualified_name":"\"It's Slow\"","kind":"section_3","ident_start":69500,"ident_end":69511,"extent_start":69496,"extent_end":70564,"ident_utf16":{"start":{"line_number":1657,"utf16_col":4},"end":{"line_number":1657,"utf16_col":15}},"extent_utf16":{"start":{"line_number":1657,"utf16_col":0},"end":{"line_number":1682,"utf16_col":0}}},{"name":"Instrument everything","fully_qualified_name":"Instrument everything","kind":"section_3","ident_start":70568,"ident_end":70589,"extent_start":70564,"extent_end":73285,"ident_utf16":{"start":{"line_number":1682,"utf16_col":4},"end":{"line_number":1682,"utf16_col":25}},"extent_utf16":{"start":{"line_number":1682,"utf16_col":0},"end":{"line_number":1738,"utf16_col":0}}},{"name":"Logging","fully_qualified_name":"Logging","kind":"section_3","ident_start":73289,"ident_end":73296,"extent_start":73285,"extent_end":73602,"ident_utf16":{"start":{"line_number":1738,"utf16_col":4},"end":{"line_number":1738,"utf16_col":11}},"extent_utf16":{"start":{"line_number":1738,"utf16_col":0},"end":{"line_number":1748,"utf16_col":0}}},{"name":"Shadow traffic","fully_qualified_name":"Shadow traffic","kind":"section_3","ident_start":73606,"ident_end":73620,"extent_start":73602,"extent_end":73968,"ident_utf16":{"start":{"line_number":1748,"utf16_col":4},"end":{"line_number":1748,"utf16_col":18}},"extent_utf16":{"start":{"line_number":1748,"utf16_col":0},"end":{"line_number":1758,"utf16_col":0}}},{"name":"Versioning","fully_qualified_name":"Versioning","kind":"section_3","ident_start":73972,"ident_end":73982,"extent_start":73968,"extent_end":74258,"ident_utf16":{"start":{"line_number":1758,"utf16_col":4},"end":{"line_number":1758,"utf16_col":14}},"extent_utf16":{"start":{"line_number":1758,"utf16_col":0},"end":{"line_number":1766,"utf16_col":0}}},{"name":"Rollouts","fully_qualified_name":"Rollouts","kind":"section_3","ident_start":74262,"ident_end":74270,"extent_start":74258,"extent_end":75007,"ident_utf16":{"start":{"line_number":1766,"utf16_col":4},"end":{"line_number":1766,"utf16_col":12}},"extent_utf16":{"start":{"line_number":1766,"utf16_col":0},"end":{"line_number":1783,"utf16_col":0}}},{"name":"Automated Control","fully_qualified_name":"Automated Control","kind":"section_3","ident_start":75011,"ident_end":75028,"extent_start":75007,"extent_end":76142,"ident_utf16":{"start":{"line_number":1783,"utf16_col":4},"end":{"line_number":1783,"utf16_col":21}},"extent_utf16":{"start":{"line_number":1783,"utf16_col":0},"end":{"line_number":1806,"utf16_col":0}}},{"name":"Feature flags","fully_qualified_name":"Feature flags","kind":"section_3","ident_start":76146,"ident_end":76159,"extent_start":76142,"extent_end":76833,"ident_utf16":{"start":{"line_number":1806,"utf16_col":4},"end":{"line_number":1806,"utf16_col":17}},"extent_utf16":{"start":{"line_number":1806,"utf16_col":0},"end":{"line_number":1821,"utf16_col":0}}},{"name":"Chaos engineering","fully_qualified_name":"Chaos engineering","kind":"section_3","ident_start":76837,"ident_end":76854,"extent_start":76833,"extent_end":77490,"ident_utf16":{"start":{"line_number":1821,"utf16_col":4},"end":{"line_number":1821,"utf16_col":21}},"extent_utf16":{"start":{"line_number":1821,"utf16_col":0},"end":{"line_number":1837,"utf16_col":0}}},{"name":"Oh no, queues","fully_qualified_name":"Oh no, queues","kind":"section_3","ident_start":77494,"ident_end":77507,"extent_start":77490,"extent_end":78644,"ident_utf16":{"start":{"line_number":1837,"utf16_col":4},"end":{"line_number":1837,"utf16_col":17}},"extent_utf16":{"start":{"line_number":1837,"utf16_col":0},"end":{"line_number":1862,"utf16_col":0}}},{"name":"Review","fully_qualified_name":"Review","kind":"section_2","ident_start":78647,"ident_end":78653,"extent_start":78644,"extent_end":79129,"ident_utf16":{"start":{"line_number":1862,"utf16_col":3},"end":{"line_number":1862,"utf16_col":9}},"extent_utf16":{"start":{"line_number":1862,"utf16_col":0},"end":{"line_number":1872,"utf16_col":0}}},{"name":"Further reading","fully_qualified_name":"Further reading","kind":"section_2","ident_start":79132,"ident_end":79147,"extent_start":79129,"extent_end":80386,"ident_utf16":{"start":{"line_number":1872,"utf16_col":3},"end":{"line_number":1872,"utf16_col":18}},"extent_utf16":{"start":{"line_number":1872,"utf16_col":0},"end":{"line_number":1894,"utf16_col":0}}},{"name":"Online","fully_qualified_name":"Online","kind":"section_3","ident_start":79153,"ident_end":79159,"extent_start":79149,"extent_end":79881,"ident_utf16":{"start":{"line_number":1874,"utf16_col":4},"end":{"line_number":1874,"utf16_col":10}},"extent_utf16":{"start":{"line_number":1874,"utf16_col":0},"end":{"line_number":1882,"utf16_col":0}}},{"name":"Trees","fully_qualified_name":"Trees","kind":"section_3","ident_start":79885,"ident_end":79890,"extent_start":79881,"extent_end":80386,"ident_utf16":{"start":{"line_number":1882,"utf16_col":4},"end":{"line_number":1882,"utf16_col":9}},"extent_utf16":{"start":{"line_number":1882,"utf16_col":0},"end":{"line_number":1894,"utf16_col":0}}}]}},"codeViewLayoutRoute":{"repo":{"id":55260959,"defaultBranch":"master","name":"distsys-class","ownerLogin":"aphyr","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2016-04-01T20:39:47.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/3748?v=4","public":true,"private":false,"isOrgOwned":false},"currentUser":null,"uploadToken":"Bx_nQDzOqbPKyAl7OAaaYZZI-GMuu1z5ZLm181WSlXiS_RHDQhNUCtrau5VEvN6vg01TA0HbWzZ5G2AOZbI6Wg","allShortcutsEnabled":false,"treeExpanded":true,"path":"README.markdown","symbolsExpanded":false,"refInfo":{"name":"master","listCacheKey":"v0:1459543355.0","canEdit":false,"currentOid":"a32c879536c1595b3ea61044fef1350c56a13349"},"helpUrl":"https://docs.github.com","githubDevUrl":null},"codeViewFileTreeLayoutRoute":{"fileTree":{"":{"items":[{"name":"schedules","path":"schedules","contentType":"directory"},{"name":"README.markdown","path":"README.markdown","contentType":"file"}],"totalCount":2}},"fileTreeProcessingTime":19.396134,"foldersToFetch":[]},"codeViewBlobLayoutRoute":{"codeLineWrapEnabled":false,"refInfo":{"name":"master","listCacheKey":"v0:1459543355.0","canEdit":false,"refType":"branch","currentOid":"a32c879536c1595b3ea61044fef1350c56a13349","canEditOnDefaultBranch":false,"fileExistsOnDefault":true},"path":"README.markdown","blob":{"copilotSWEAgentEnabled":false,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/aphyr/distsys-class/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"README.markdown","displayUrl":"https://github.com/aphyr/distsys-class/blob/master/README.markdown?raw=true","headerInfo":{"blobSize":"78.5 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"a13267b","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Faphyr%2Fdistsys-class%2Fblob%2Fmaster%2FREADME.markdown","isCSV":false,"isRichtext":true,"lineInfo":{"truncatedLoc":"1894","truncatedSloc":"1661"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"isIssueTemplate":false,"isDiscussionTemplate":false,"language":"Markdown","languageID":222,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/aphyr/distsys-class/blob/master/README.markdown","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/aphyr/distsys-class/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/aphyr/distsys-class/raw/refs/heads/master/README.markdown","renderImageOrRaw":false,"shortPath":null,"symbolsEnabled":true,"tabSize":4,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null},"copilotInfo":null,"copilotAccessAllowed":false,"copilotSpacesEnabled":false,"modelsAccessAllowed":false,"modelsRepoIntegrationEnabled":false,"isMarketplaceEnabled":true},"codeViewBlobLayoutRoute.StyledBlob":{"rawLines":["# An Introduction to Distributed Systems","","Copyright Kyle Kingsbury \u0026 Jepsen, LLC.","","This outline accompanies a 16 to 32 hour [overview class on distributed systems","fundamentals](https://jepsen.io/training.html). The course aims to introduce","software engineers to the practical basics of distributed systems, through","lecture and discussion, and optional","[labwork](https://github.com/jepsen-io/maelstrom). Participants will gain an","intuitive understanding of key distributed systems terms, an overview of the","algorithmic landscape, and explore production concerns.","","## Intro","","- My name is Kyle Kingsbury"," - Pronouns: he/him"," - Email: aphyr@jepsen.io"," - This outline: https://github.com/aphyr/distsys-class","- I've worked on distributed systems from several perspectives"," - 2003--2009: sysadmin, network operations, web developer"," - 2009--2014: backend engineer, database wrangler at various web startups"," - 2014--now: distributed systems safety testing (Jepsen)","- This class is aimed at practitioners"," - \"Things I Wish I Would Have Known\""," - Backend engineers"," - Frontend engineers"," - Ops folks"," - Product managers","- This class aims to prepare you to write, operate, and use distributed systems"," - First half: theoretical framework"," - A map of how concepts fit together"," - Not about memorizing everything; it's about knowing where to look"," - Establish a shared language"," - To talk with your peers"," - To read a paper"," - To evaluate a system's claims"," - Understand fundamental principles"," - Time, ordering, nodes, networks, faults, liveness, safety"," - Classes of algorithms"," - What they can and can't do, when to apply each"," - Second half: practical concerns"," - A grab bag of design patterns"," - Illustrative anecdotes"," - Computers were, in fact, a mistake"," - What *isn't* in the papers?"," - Finally, production concerns"," - Deployment, debugging, monitoring","- Class logistics"," - This class is what you make of it"," - We can go as deep as you want, or skip over familiar ground"," - Jump in with questions, challenges, ideas at any time"," - We'll take ~10-minute breaks every hour or so, plus lunch"," - If you need to duck out for whatever reason, that's fine!"," - Outages, kids, pets, whatever"," - The outline on GitHub can help fill in the gaps"," - You can ask me during class or a break about something you missed!"," - I'll ask for your thoughts often, but you can also jump in whenever"," - If you *don't* want to be called on, that's cool--please let me know"," - If you think of something after class, you can email me!"," - Or write it down for the next day's discussion"," - Recording is prohibited"," - Teaching is my livelihood"," - And this outline is free!","- If we're doing labs:"," - There are labs for this class!"," - Mostly in the middle"," - Remind folks of the prereq: [Getting Ready](https://github.com/jepsen-io/maelstrom/blob/main/doc/01-getting-ready/index.md)","- If we're remote:"," - You can turn off video whenever you like"," - But please, if *some* people can be on video, that's really helpful"," - Seeing faces helps me know whether the lecture is working for you!"," - For the labs, we'll be doing a shared tmux session","- OK, let's get going!","","## What makes a thing distributed?","","Lamport, 1987:","","\u003e A distributed system is one in which the failure of a computer","\u003e you didn't even know existed can render your own computer","\u003e unusable.","","- First glance: \\*nix boxen in our colo, running processes communicating via"," TCP or UDP."," - Or boxes in EC2, Rackspace, etc"," - Maybe communicating over InfiniBand"," - Separated by inches and a LAN"," - Or by kilometers and the internet","- Most mobile apps are also taking part in a distributed system"," - Communicating over a truly awful network"," - Same goes for desktop web browsers"," - It's not just servers--it's clients too!","- More generally: distributed systems are"," - Made up of parts which interact"," - Slowly"," - And often unreliably"," - Whatever those mean for you","- So also:"," - Redundant CPUs in an airplane"," - ATMs and Point-of-Sale terminals"," - Space probes"," - Paying bills"," - Doctors making referrals"," - Trying to make plans via text message"," - Every business meeting ever","","## Nodes and networks","","- We call each part of a distributed system a *node*"," - Also known as *processes*, *agents*, or *actors*","","### Nodes","","- Characteristic latency"," - Operations inside a node are \"fast\""," - Operations between nodes are \"slow\""," - What's fast or slow depends on what the system does","- Nodes are reliable"," - Fail as a unit"," - You know when problems occur"," - State is coherent"," - State transitions occur in a nice, orderly fashion"," - Typically modeled as some kind of single-threaded state machine","- A node could *itself* be a distributed system"," - But so long as that system as a whole provides \"fast, coherent\""," operations, we can treat it as a single node.","- Formal models for processes"," - [Actor model](https://dspace.mit.edu/handle/1721.1/6952)"," - [Communicating Sequential Processes](https://www.cs.cmu.edu/~crary/819-f09/Hoare78.pdf)"," - [Pi-calculus](https://golem.ph.utexas.edu/category/2009/08/the_pi_calculus.html)"," - [Ambient calculus](http://lucacardelli.name/Talks/1998-03-30%20Mobile%20Ambients%20%28ETAPS%29.pdf)","- Formal models for node failure"," - Crash-stop"," - Crash-recover"," - Crash-amnesia"," - Byzantine","","### Networks as message flows","","- Nodes interact via a *network*"," - Humans interact via spoken words"," - Particles interact via fields"," - Computers interact via IP, or UDP, or SCTP, or ...","- We model those interactions as discrete *messages* sent between nodes","- Messages take *time* to propagate"," - This is the \"slow\" part of the distributed system"," - We call this \"latency\"","- Messages can often be lost"," - This is another \"unreliable\" part of the distributed system","- Network is rarely homogenous"," - Some links slower/smaller/more-likely-to-fail than others","","### Causality diagrams","","- We can represent the interaction of nodes and the network as a diagram"," - Time flows left-to-right, or top-to-bottom"," - Nodes are lines in the direction of time (because they stay in place)"," - Messages as slanted paths *connecting* nodes","","### Synchronous networks","","- Nodes execute in lockstep: time between node steps is always 1.","- Message delay is bounded","- Effectively a perfect global clock","- Easy to prove stuff about"," - You probably don't have one","","### Semi-synchronous networks","","- Like synchronous, but the clock is only approximate, e.g. in [c, 1]","","### Asynchronous networks","","- Execute independently, whenever: step time is anywhere in [0, 1]","- Unbounded message delays","- No global clocks","- Weaker than semi- or synchronous networks"," - Implies certain algorithms can't be as efficient"," - Implies certain algorithms are *impossible*"," - See e.g. Attiya \u0026 Mavronicolas, \"Efficiency of Semi-Synchronous vs"," Asynchronous Networks\"","- IP networks are definitely asynchronous"," - But *in practice* the really pathological stuff doesn't happen"," - Most networks recover in seconds to weeks, not \"never\""," - Conversely, human timescales are on the orders of seconds to weeks"," - So we can't pretend the problems don't exist","","## When networks go wrong","","- Asynchronous networks are allowed to"," - Duplicate"," - Delay"," - Drop"," - Reorder","- Drops and delays are indistinguishable","- Byzantine networks are allowed to mess with messages *arbitrarily*"," - Including rewriting their content"," - They mostly don't happen in real networks"," - Mostly"," - https://www.pagerduty.com/blog/the-discovery-of-apache-zookeepers-poison-packet/"," - https://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19","","## Low level protocols","","### TCP","","- TCP *works*. Use it."," - Not perfect; you can go faster"," - But you'll know when this is the case","- In practice, TCP prevents duplicates and reorders in the context of a single"," TCP conn"," - But you're probably gonna open more than one connection"," - If for no other reason than TCP conns eventually fail"," - And when that happens, you'll either a.) have missed messages or b.) retry"," - You can *reconstruct* an ordering by encoding your own sequence numbers on"," top of TCP","","### UDP","","- Same addressing rules as TCP, but no stream invariants","- Lots of people want UDP \"for speed\""," - Don't consider that routers and nodes can and will arbitrarily drop packets"," - Don't consider that their packets *will* be duplicated"," - And reordered"," - \"But at least it's unbiased right?\""," - WRONG!"," - This causes all kinds of havoc in, say, metrics collection"," - And debugging it is *hard*"," - TCP gives you flow control and repacks logical messages into packets"," - You'll need to re-build flow-control and backpressure"," - TLS over UDP is a thing, but tough","- UDP is really useful where TCP FSM overhead is prohibitive"," - Memory pressure"," - Lots of short-lived conns and socket reuse","- Especially useful where best-effort delivery maps well to the system goals"," - Voice calls: people will apologize and repeat themselves"," - Games: stutters and lag, but catch up later"," - Higher-level protocols impose sanity on underlying chaos","","","## Clocks","","- When a system is split into independent parts, we still want some kind of"," *order* for events","- Clocks help us order things: first this, THEN that","","### Wall Clocks","","- In theory, the operating system clock gives you a partial order on system events"," - Caveat: NTP is probably not as good as you think"," - Caveat: Definitely not well-synced between nodes"," - Caveat: Hardware can drift"," - Multiple reports of supermicro TSCs causing system clock to run 26ms/s fast"," - https://groups.google.com/forum/#!search/Supermicro$20SYS-1029UX-LL1-S16$20system$20clock$20running$20too$20fast/mechanical-sympathy/oG9vLZVYjVA/DU-T9QpBAgAJ"," - Caveat: By *centuries*"," - NTP might not care"," - http://rachelbythebay.com/w/2017/09/27/2153/"," - Caveat: NTP can still jump the clock backwards (default: delta \u003e 128 ms)"," - https://www.eecis.udel.edu/~mills/ntp/html/clock.html"," - Caveat: POSIX time is not monotonic by *definition*"," - Cloudflare 2017: Leap second at midnight UTC meant time flowed backwards"," - At the time, Go didn't offer access to CLOCK_MONOTONIC"," - Computed a negative duration, then fed it to rand.int63n(), which paniced"," - Caused DNS resolutions to fail: 1% of HTTP requests affected for several hours"," - https://blog.cloudflare.com/how-and-why-the-leap-second-affected-cloudflare-dns/"," - Caveat: The timescales you want to measure may not be attainable"," - Caveat: Threads can sleep"," - Caveat: Runtimes can sleep"," - Caveat: OS's can sleep"," - Caveat: \"Hardware\" can sleep"," - Caveat: The hypervisor can lie to you"," - By 16+ seconds in a 15 minute period!?"," - https://gist.github.com/sandfox/32e749b5eac861c93f1bbeb8782ae8fd","- Just don't.","- At least OS monotonic clocks are monotonic, right?"," - oh no: https://github.com/rust-lang/rust/blob/eed12bcd0cb281979c4c9ed956b9e41fda2bfaeb/src/libstd/time.rs#L201-L232"," - Use CLOCK_BOOTTIME, which accounts for VM pauses","","### Lamport Clocks","","- Lamport 1977: \"Time, Clocks, and the Ordering of Events in a Distributed System\""," - One clock per process"," - Increments monotonically with each state transition: `t' = t + 1`"," - Included with every message sent"," - `t' = max(t, t_msg + 1)`","- If we have a total ordering of processes, we can impose a total order on"," events"," - But that order could be pretty unintuitive","","### Vector Clocks","","- Generalizes Lamport clocks to a vector of all process clocks","- `t_i' = max(t_i, t_msg_i)`","- For every operation, increment that process' clock in the vector","- Provides a partial causal order"," - A \u003c B iff all A_i \u003c= B_i, and at least one A_i \u003c B_i"," - Specifically, given a pair of events, we can determine causal relationships"," - A in causal past of B implies A \u003c B"," - B in causal past of A implies B \u003c A"," - Independent otherwise","- Pragmatically: the past is shared; the present is independent"," - Only \"present\", independent states need to be preserved"," - Ancestor states can be discarded"," - Lets us garbage-collect the past","- O(processes) in space"," - Requires coordination for GC"," - Or sacrifice correctness and prune old vclock entries","- Variants"," - Dotted Version Vectors - for client/server systems, orders *more* events"," - Interval Tree Clocks - for when processes come and go","","### GPS \u0026 Atomic Clocks","","- Much better than NTP"," - Globally distributed total orders on the scale of milliseconds"," - Promote an asynchronous network to a semi-synchronous one"," - Unlocks more efficient algorithms","- Only people with this right now are Google"," - Spanner: globally distributed strongly consistent transactions"," - And they're not sharing","- More expensive than you'd like"," - Several hundred per GPS receiver"," - Atomic clocks for local corroboration: $$$$?"," - Probably want multiple types of GPS clock"," - https://rachelbythebay.com/w/2015/09/07/noleap/"," - Confusing vendor checkbox which applied UTC corrections to GPS time"," - I don't know who's doing it yet, but I'd bet datacenters in the"," future will offer dedicated HW interfaces for bounded-accuracy time.","","","## Review","","We've covered the fundamental primitives of distributed systems. Nodes","exchange messages through a network, and both nodes and networks can fail in","various ways. Protocols like TCP and UDP give us primitive channels for","processes to communicate, and we can order events using clocks. Now, we'll","discuss some high-level *properties* of distributed systems.","","","","","## Availability","","- Availability is basically the fraction of attempted operations which succeed.","","### Total availability","","- Naive: every operation succeeds","- In consistency lit: every operation on a non-failing node succeeds"," - Nothing you can do about the failing nodes","","### Sticky availability","","- Every operation against a non-failing node succeeds"," - With the constraint that clients always talk to the same nodes","","### High availability","","- Better than if the system *weren't* distributed.","- e.g. tolerant of up to f failures, but no more","- Maybe some operations fail","","### Majority available","","- Operations succeed *if* they occur on a node which can communicate"," with a majority of the cluster","- Operations against minority components may fail","","### Quantifying availability","","- We talk a lot about \"uptime\""," - Are systems up if nobody uses them?"," - Is it worse to be down during peak hours?"," - Can measure \"fraction of requests satisfied during a time window\""," - Then plot that fraction over windows at different times"," - Timescale affects reported uptime","- Apdex"," - Not all successes are equal"," - Classify operations into \"OK\", \"meh\", and \"awful\""," - Apdex = P(OK) + P(meh)/2"," - Again, can report on a yearly basis"," - \"We achieved 99.999 apdex for the year\""," - And on finer timescales!"," - \"Apdex for the user service just dropped to 0.5; page ops!\"","- Ideally: integral of happiness delivered by your service?","","","## Consistency","","- A consistency model is the set of \"safe\" histories of events in the system","","### Monotonic Reads","","- Once I read a value, any subsequent read will return that state or later values","","### Monotonic Writes","","- If I make a write, any subsequent writes I make will take place *after* the"," first write","","### Read Your Writes","","- Once I write a value, any subsequent read I perform will return that write"," (or later values)","","### Writes Follow Reads","","- Once I read a value, any subsequent write will take place after that read","","### Causal consistency","","- Suppose operations can be linked by a DAG of causal relationships"," - A write that follows a read, for instance, is causally related"," - Assuming the process didn't just throw away the read data"," - Operations not linked in that DAG are *concurrent*","- Constraint: before a process can execute an operation, all its precursors"," must have executed on that node","- Concurrent ops can be freely reordered","","### Sequential consistency","","- Like causal consistency, constrains possible orders","- All operations appear to execute atomically","- Every process agrees on the order of operations"," - Operations from a given process always occur in order"," - But nodes can lag behind","","### Linearizability","","- All operations appear to execute atomically","- Every process agrees on the order of operations","- Every operation appears to take place *between* its invocation and completion"," times","- Real-time, external constraints let us build very strong systems","","### Transactional Models","","- Serializable: all operations (transactions) appear to execute atomically"," - In some order"," - No constraints on what that order is"," - Perfectly okay to read from the past, for instance","- Adya 1999: Weak Consistency: A Generalized Theory and Optimistic"," Implementations for Distributed Transactions"," - Objects have a total version order"," - x0 \u003c\u003c x1 \u003c\u003c x2"," - This is an abstract requirement; DB may not know what the version order is"," - Dependencies between transactions"," - write-read (wr): T1 writes x1, T2 reads x1"," - write-write (ww): T1 writes x1, T2 writes x2"," - read-write (rw): T1 reads x1, T2 writes x2"," - Phenomena are (mostly) cycles made of these edges"," - G0 (write cycle): cycle in ww"," - G1a (aborted read): committed txn reads version written by aborted txn"," - G1b (intermediate read): read non-final write of x by some other txn"," - G1c (cyclic information flow): cycle in ww U wr"," - G2 (anti-dependency cycle): cycle in ww U wr U rw"," - G2-item: G2 without predicates"," - G-single: G2 with only one rw"," - etc. etc."," - Isolation levels prevent these phemomena"," - Read Uncommitted: no G0"," - Read Committed: no G0, G1"," - Repeatable Read: no G0, G1, G2-item"," - SI: No g0, G1, G-single"," - It's a little more subtle than this, but we generally don't have time"," for G-nonadjacent"," - Serializable: no G0, G1, G2"," - Can augment graphs with process or realtime edges"," - Strong X means \"X, plus order of txns in real time\""," - Strong Session X means \"X, plus order of txns in each session\""," - e.g. Strong Serializable: no cycle in ww U wr U rw U realtime","- Alternate route: ANSI SQL's ACID isolation levels are weird"," - Basically codified the effects of existing vendor implementations"," - Definitions in the spec are ambiguous"," - Each ANSI SQL isolation level prohibits a weird phenomenon"," - Read Uncommitted"," - Prevents P0: *dirty writes*"," - w1(x) ... w2(x)"," - Can't write over another transaction's data until it commits"," - Can read data while a transaction is still modifying it"," - Can read data that will be rolled back"," - Read Committed"," - Prevents P1: *dirty reads*"," - w1(x) ... r2(x)"," - Can't read a transaction's uncommitted values"," - Repeatable Read"," - Prevents P2: *fuzzy reads*"," - r1(x) ... w2(x)"," - Once a transaction reads a value, it won't change until the transaction"," commits"," - Serializable"," - Prevents P3: *phantoms*"," - Given some predicate P"," - r1(P) ... w2(y in P)"," - Once a transaction reads a set of elements satisfying a query, that"," set won't change until the transaction commits"," - Not just values, but *which values even would have participated*."," - Cursor Stability"," - Transactions have a set of cursors"," - A cursor refers to an object being accessed by the transaction"," - Read locks are held until cursor is removed, or commit"," - At commit time, cursor is upgraded to a writelock"," - Prevents lost-update"," - Snapshot Isolation"," - Transactions always read from a snapshot of committed data, taken before"," the transaction begins"," - Commit can only occur if no other committed transaction with an"," overlapping [start..commit] interval has written to any of the objects"," *we* wrote"," - First-committer-wins","","### Does any of this actually matter?","","- Real world just isn't that concurrent","- Plenty of companies get by on Read Committed","- But *malicious* attackers can *induce* concurrency"," - Flexcoin"," - Bitcoin exchange which allowed users to create money by shuffling between accounts"," - Attacked in 2014, 365,000 GBP stolen"," - Exchange collapsed altogether"," - Poloniex"," - Concurrent withdrawals were improperly isolated, allowing users to overspend"," - Safety audits didn't notice negative balances"," - 12.3% of exchange funds stolen; loss spread among users"," - [Warszawski \u0026 Bailis 2017: ACIDRain](http://www.bailis.org/papers/acidrain-sigmod2017.pdf)"," - Automated identification of consistency violation in web apps"," - e.g. Buy one gift card, then spend it an unlimited number of times"," - e.g. Buy a pen, add a laptop to cart during checkout, get a free laptop"," - Vulnerabilities found in over 50% of all eCommerce web sites"," - Weak DB isolation defaults"," - Improper use of transactional scope"," - Failing to use any transactions whatsoever"," - [Chase Bank's credit card rewards system](https://chadscira.com/post/5fa269d46142ac544e013d6e/DISCLOSURE-Unlimited-Chase-Ultimate-Rewards-Points)"," - Concurrent transfers between balances allowed the creation of $70,000 USD"," in travel vouchers."," - Redeemable for cash!","","## Tradeoffs","","- Ideally, we want total availability and linearizability","- Consistency requires coordination"," - If every order is allowed, we don't need to do any work!"," - If we want to disallow some orders of events, we have to exchange messages","- Coordinating comes (generally) with costs"," - More consistency is slower"," - More consistency is more intuitive"," - More consistency is less available","","### Availability and Consistency","","- CAP Theorem: Linearizability OR total availability","- But wait, there's more!"," - Bailis 2014: Highly Available Transactions: Virtues and Limitations"," - Other theorems disallow totally or sticky available..."," - Strong serializable"," - Serializable"," - Repeatable Read"," - Cursor Stability"," - Snapshot Isolation"," - You can have *sticky* available..."," - Causal"," - PRAM"," - Read Your Writes"," - You can have *totally* available..."," - Read Uncommitted"," - Read Committed"," - Monotonic Atomic View"," - Writes Follow Reads"," - Monotonic Reads"," - Monotonic Writes","","","### Harvest and Yield","","- Fox \u0026 Brewer, 1999: Harvest, Yield, and Scalable Tolerant Systems"," - Yield: probability of completing a request"," - Harvest: fraction of data reflected in the response"," - Examples"," - Node faults in a search engine can cause some results to go missing"," - Updates may be reflected on some nodes but not others"," - Consider an AP system split by a partition"," - You can write data that some people can't read"," - Streaming video degrades to preserve low latency"," - This is not an excuse to violate your safety invariants"," - Just helps you quantify how much you can *exceed* safety invariants"," - e.g. \"99% of the time, you can read 90% of your prior writes\""," - Strongly dependent on workload, HW, topology, etc"," - Can tune harvest vs yield on a per-request basis"," - \"As much as possible in 10ms, please\""," - \"I need everything, and I understand you might not be able to answer\"","","### Hybrid systems","","- So, you've got a spectrum of choices!"," - Chances are different parts of your infrastructure have different needs"," - Pick the weakest model that meets your constraints"," - But consider probabilistic bounds; visibility lag might be prohibitive"," - See Probabilistically Bounded Staleness in Dynamo Quorums","- Not all data is equal"," - Big data is usually less important"," - Small data is usually critical"," - Linearizable user ops, causally consistent social feeds","","### Review","","Availability is a measure of how often operations succeed. Consistency models","are the rules that govern what operations can happen and when. Stronger","consistency models generally come at the cost of performance and availability.","Next, we'll talk about different ways to build systems, from weak to strong","consistency.","","","## Avoid Consensus Wherever Possible","","### CALM conjecture","","- Consistency As Logical Monotonicity"," - If you can prove a system is logically monotonic, it is coordination free"," - What the heck is \"coordination\""," - For that matter, what's \"monotonic\"?","- Monotonicity, informally, is retraction-free"," - Deductions from partial information are never invalidated by new information"," - Both relational algebra and Datalog without negation are monotone","- Ameloot, et al, 2011: Relational transducers for declarative networking"," - Theorem which shows coordination-free networks of processes unaware of the"," network extent can compute only monotone queries in Datalog"," - This is not an easy read"," - \"Coordination-free\" doesn't mean no communication"," - Algo succeeds even in face of arbitrary horizontal partitions","- In very loose practical terms"," - Try to phrase your problem such that you only *add* new facts to the system"," - When you compute a new fact based on what's currently known, can you ensure"," that fact will never be retracted?"," - Consider special \"sealing facts\" that mark a block of facts as complete"," - These \"grow-only\" algorithms are usually easier to implement"," - Likely tradeoff: incomplete reads","- Bloom language"," - Unordered programming with flow analysis"," - Can tell you where coordination *would* be required","","","### Gossip","","- Message broadcast system","- Useful for cluster management, service discovery, health, sensors, CDNs, etc","- Generally weak consistency / high availability","- Global broadcast"," - Send a message to every other node"," - O(nodes)","- Mesh networks"," - Epidemic models"," - Relay to your neighbors"," - Propagation times on the order of max-free-path","- Spanning trees"," - Instead of a mesh, use a tree"," - Hop up to a connector node which relays to other connector nodes"," - Reduces superfluous messages"," - Reduces latency"," - Plumtree (Leit ̃ao, Pereira, \u0026 Rodrigues, 2007: Epidemic Broadcast Trees)","- Push-Sum"," - Kempe, Dobra, \u0026 Gehrke - Gossip-Based Computation of Aggregate Information"," - Sum inputs from everyone you've received data from"," - Broadcast that to a random peer"," - Extensions for minima, maxima, means"," - Helpful for live metrics, rate limiting, routing, identifying cluster"," hotspots","","### CRDTs","","- Order-free datatypes that converge"," - Counters, sets, maps, etc","- Tolerate dupes, delays, and reorders","- Unlike sequentially consistent systems, no \"single source of truth\"","- But unlike naive eventually consistent systems, never *lose* information"," - Unless you explicitly make them lose information"," - We call this property \"coalescence\"","- Works well in highly-available systems"," - Web/mobile clients"," - Dynamo"," - Gossip","- INRIA: Shapiro, Preguiça, Baquero, Zawirski, 2011: \"A comprehensive study of"," Convergent and Commutative Replicated Data Types\""," - Composed of a data type X and a merge function m, which is:"," - Associative: m(x1, m(x2, x3)) = m(m(x1, x2), x3)"," - Commutative: m(x1, x2) = m(x2, x1)"," - Idempotent: m(x1, x1) = m(x1)","- Easy to build. Easy to reason about. Gets rid of all kinds of headaches."," - Did communication fail? Just retry! It'll converge!"," - Did messages arrive out of order? It's fine!"," - How do I synchronize two replicas? Just merge!","- Downsides"," - Some algorithms *need* order and can't be expressed with CRDTs"," - Reads may be arbitrarily stale"," - Higher space costs","","### HATs","","- Bailis, Davidson, Fekete, et al, 2013: \"Highly Available Transactions,"," Virtues and Limitations\""," - Guaranteed responses from any replica"," - Low latency (1-3 orders of magnitude faster than serializable protocols!)"," - Read Committed"," - Monotonic Atomic View"," - Excellent for commutative/monotonic systems"," - Foreign key constraints for multi-item updates"," - Limited uniqueness constraints"," - Can ensure convergence given arbitrary finite delay (\"eventual consistency\")"," - Good candidates for geographically distributed systems"," - Probably best in concert with stronger transactional systems"," - See also: COPS, Swift, Eiger, Calvin, etc","","","## Fine, We Need Consensus, What Now?","","- The consensus problem:"," - Three process types"," - Proposers: propose values"," - Acceptors: choose a value"," - Learners: read the chosen value"," - Classes of acceptors"," - N acceptors total"," - F acceptors allowed to fail"," - M malicious acceptors"," - Three invariants:"," - Nontriviality: Only values proposed can be learned"," - Safety: At most one value can be learned"," - Liveness: If a proposer p, a learner l, and a set of N-F acceptors are"," non-faulty and can communicate with each other, and if p proposes a"," value, l will eventually learn a value.","","- Whole classes of systems are *equivalent* to the consensus problem"," - So any proofs we have here apply to those systems too"," - Lock services"," - Ordered logs"," - Replicated state machines","","- FLP tells us consensus is impossible in asynchronous networks"," - Kill a process at the right time and you can break *any* consensus algo"," - True but not as bad as you might think"," - Realistically, networks work *often enough* to reach consensus"," - Moreover, FLP assumes deterministic processes"," - Real computers *aren't* deterministic"," - Ben-Or 1983: \"Another Advantage of free choice\""," - Nondeterministic algorithms *can* achieve consensus","","- Lamport 2002: tight bounds for asynchronous consensus"," - With at least two proposers, or one malicious proposer, N \u003e 2F + M"," - \"Need a majority\""," - With at least 2 proposers, or one malicious proposer, it takes at least 2"," message delays to learn a proposal.","","- This is a pragmatically achievable bound"," - In stable clusters, you can get away with only a single round-trip to a"," majority of nodes."," - More during cluster transitions.","","","### Paxos","","- Paxos is the Gold Standard of consensus algorithms"," - Lamport 1989 - The Part Time Parliament"," - Written as a description of an imaginary Greek democracy"," - Lamport 2001 - Paxos Made Simple"," - \"The Paxos algorithm for implementing a fault-tolerant distributed system"," has been regarded as difficult to understand, perhaps because the"," original presentation was Greek to many readers [5]. In fact, it is among the"," simplest and most obvious of distributed algorithms... The last section"," explains the complete Paxos algorithm, which is obtained by the straightforward"," application of consensus to the state machine approach for building a"," distributed system—an approach that should be well-known, since it is the"," subject of what is probably the most often-cited article on the theory of"," distributed systems [4].\""," - Google 2007 - Paxos Made Live"," - Notes from productionizing Chubby, Google's lock service"," - Van Renesse 2011 - Paxos Made Moderately Complex"," - Turns out you gotta optimize"," - Also pseudocode would help"," - A page of pseudocode -\u003e several thousand lines of C++","- Provides consensus on independent proposals","- Typically deployed in majority quorums, 5 or 7 nodes","- Several optimizations"," - Multi-Paxos"," - Fast Paxos"," - Generalized Paxos"," - It's not always clear which of these optimizations to use, and which"," can be safely combined"," - Each implementation uses a slightly different flavor"," - Paxos is really more of a *family* of algorithms than a well-described"," single entity","- Used in a variety of production systems"," - Chubby"," - Cassandra"," - Riak"," - FoundationDB"," - WANdisco SVN servers","- New research: Paxos quorums need not be majority: can optimize for fast phase-2 quorums [Howard, Malkhi, and Spiegelman](https://arxiv.org/abs/1608.06696)."," - We're not sure how to USE this yet"," - Durability still requires distribution","","### ZAB","","- ZAB is the Zookeeper Atomic Broadcast protocol","- Junqueira, Reed, and Serafini 2011 - Zab: High-performance broadcast for"," primary-backup systems","- Differs from Paxos","- Provides sequential consistency (linearizable writes, lagging ordered reads)"," - Useful because ZK clients typically want fast local reads"," - But there's also a SYNC command that guarantees real-time visibility"," - (SYNC + op) allows linearizable reads as well","- Again, majority quorum, 5 or 7 nodes","","### Humming Consensus","","- Metadata store for managing distributed system reconfiguration","- Looks a little like CORFU's replicated log","- See also: chain replication","","### Viewstamped Replication","","- Presented as a replication protocol, but also a consensus algorithm","- Transaction processing plus a view change algorithm","- Majority-known values are guaranteed to survive into the future","- I'm not aware of any production systems, but I'm sure they're out there","- Along with Paxos, inspired Raft in some ways","","### Raft","","- Ongaro \u0026 Ousterhout 2014 - In Search of an Understandable Consensus Algorithm","- Lamport says it's easy, but we still have trouble grokking Paxos"," - What if there were a consensus algorithm we could actually understand?","- Paxos approaches independent decisions when what we *want* is state machines"," - Maintains a replicated *log* of state machine transitions instead","- Also builds in cluster membership transitions, which is *key* for real systems","- Very new, but we have a Coq proof of the core algorithm","- Can be used to write arbitrary sequential or linearizable state machines"," - RethinkDB"," - etcd"," - Consul","","## What About Transactions?","","- Iterated consensus gives us agreement on a single total order of operations","- Unnecessary blocking betwixt transactions which *could* execute independently","- How do we improve performance?","- [Distributed Transaction Architectures](https://aphyr.com/media/talks/2019/distributed-transaction-architectures.pdf)","- Single-writer"," - All updates go through a single queue, readers execute on snapshots"," - Usually involves some kind of persistent data structure"," - Serializable to strict-1SR"," - See [Datomic](https://docs.datomic.com/on-prem/architecture.html)","- OK but multiple writers?"," - In general, have several shards, each running a consensus-backed FSM"," - Some kind of protocol for cross-shard txns","- Independent shards"," - A sort of halfway-step to general-purpose transactions"," - Disallow cross-shard txns"," - Just run a bunch of independent consensus FSMs"," - Can add a single global consensus group for cross-shard transactions"," - Limited throughput though!"," - See [VoltDB](https://docs.voltdb.com/UsingVoltDB/IntroHowVoltDBWorks.php)","- [Percolator](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf)"," - Snapshot isolation over linearizable shards"," - Time Stamp Oracle assigns sequential txn timestamps (using consensus)"," - Read timestamp, read from leaders, prewrite, commit timestamp, commit, finalize"," - 14 network hops, potentially all cross-DC"," - See [TiDB](https://tikv.org/deep-dive/distributed-transaction/percolator/)","- [Spanner](https://static.googleusercontent.com/media/research.google.com/en//archive/spanner-osdi2012.pdf)"," - \"External consistency\" (strict-1SR?)"," - Speed up timestamp assignment by using GPS+Atomic clocks"," - Basically 2PC over Paxos groups"," - Locks on Paxos leaders"," - Pick one Paxos group to serve as the commit record for entire txn"," - Fixed latency floor to ensure timestamp monotonicity"," - See [Yugabyte DB](https://blog.yugabyte.com/distributed-postgresql-on-a-google-spanner-architecture-storage-layer/)"," - See [CockroachDB](https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/)","- [Calvin](http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf)"," - Order transactions in a log using consensus"," - Shard log for arbitrarily high throughput"," - Periodically seal log windows and apply transactions to shards"," - Application requires no communication!"," - Strict-1SR"," - 1 inter-DC round trip, more hops for local comms"," - Scalable throughput"," - Minimum latency floor"," - Txns must be pure, expressed up-front"," - Could be made interactive with extensions to protocol"," - See [Fauna](https://fauna.com/)","","## Review","","Systems which only add facts, not retract them, require less coordination to","build. We can use gossip systems to broadcast messages to other processes,","CRDTs to merge updates from our peers, and HATs for weakly isolated","transactions. Serializability and linearizability require *consensus*, which we","can obtain through Paxos, ZAB, VR, or Raft. Now, we'll talk about different","*scales* of distributed systems.","","","","## Characteristic latencies","","- Latency is *never* zero"," - Bandwidth goes up and up but we're bumping up against the physical limits"," of light and electrons"," - Latency budget shapes your system design"," - How many network calls can you afford?","- Different kinds of systems have different definitions of \"slow\""," - Different goals"," - Different algorithms","","### Multicore systems","","- Multicore (and especially NUMA) architectures are sort of like a distributed system"," - Nodes don't fail pathologically, but message exchange is slow!"," - Synchronous network provided by a bus (e.g. Intel QPI)"," - Whole complicated set of protocols in HW \u0026 microcode to make memory look"," sane"," - Non-temporal store instructions (e.g. MOVNTI)","- They provide abstractions to hide that distribution"," - MFENCE/SFENCE/LFENCE"," - Introduce a serialization point against load/store instructions"," - Characteristic latencies: ~100 cycles / ~30 ns"," - Really depends on HW, caches, instructions, etc"," - CMPXCHG Compare-and-Swap (sequentially consistent modification of memory)"," - LOCK"," - Lock the full memory subsystem across cores!","- But those abstractions come with costs"," - Hardware lock elision may help but is nascent"," - Blog: Mechanical Sympathy"," - Avoid coordination between cores wherever possible"," - Context switches (process or thread!) can be expensive"," - Processor pinning can really improve things"," - When writing multithreaded programs, try to divide your work into"," independent chunks"," - Try to align memory barriers to work unit boundaries"," - Allows the processor to cheat as much as possible within a work unit"," - See Danica Porobic, 2016: [High Performance Transaction Processing on Non-Uniform Hardware Topologies](https://infoscience.epfl.ch/record/219117/files/EPFL_TH7023.pdf)","","### Local networks","","- You'll often deploy replicated systems across something like an ethernet LAN","- Message latencies can be as low as 100 micros"," - But across any sizable network (EC2), expect low millis"," - Sometimes, packets could be delayed by *five minutes*"," - Plan for this","- Network is within an order of mag compared to uncached disk seeks"," - Or faster, in EC2"," - EC2 disk latencies can routinely hit 20ms"," - 200ms?"," - *20,000* ms???"," - Because EBS is actually other computers"," - LMAO if you think anything in EC2 is real"," - Wait, *real disks do this too*?"," - What even are IO schedulers?","- But network is waaaay slower than memory/computation"," - If your aim is *throughput*, work units should probably take longer than a"," millisecond"," - But there are other reasons to distribute"," - Sharding resources"," - Isolating failures","","### Geographic replication","","- You deploy worldwide for two reasons"," - End-user latency"," - Humans can detect ~10ms lag, will tolerate ~100ms"," - SF--Denver: 50ms"," - SF--Tokyo: 100 ms"," - SF--Madrid: 200 ms"," - Only way to beat the speed of light: move the service closer"," - Disaster recovery"," - Datacenter power is good but not perfect"," - Hurricanes are a thing"," - Entire Amazon regions can and will fail"," - Yes, regions, not AZs","- Minimum of 1 round-trip for consensus"," - Maybe as bad as 4 rounds"," - Maybe 4 rounds all the time if you have a bad Paxos impl (e.g. Cassandra)"," - So if you do Paxos between datacenters, be ready for that cost!"," - Because the minimum latencies are higher than users will tolerate"," - Cache cache cache"," - Queue writes and relay asynchronously"," - Consider reduced consistency guarantees in exchange for lower latency"," - CRDTs can always give you safe local writes"," - Causal consistency and HATs can be good calls here","- What about strongly consistent stuff?"," - Chances are a geographically distributed service has natural planes of"," cleavage"," - EU users live on EU servers; US users live on US servers"," - Use consensus to migrate users between datacenters"," - Pin/proxy updates to home datacenter"," - Which is hopefully the closest datacenter!"," - But maybe not! I believe Facebook still pushes all writes through 1 DC!"," - Where sequential consistency is OK, cache reads locally!"," - You probably leverage caching in a single DC already","","### Review","","We discussed three characteristic scales for distributed systems: multicore","processors coupled with a synchronous network, computers linked by a LAN, and","datacenters linked by the internet or dedicated fiber. CPU consequences are","largely performance concerns: knowing how to minimize coordination. On LANs,","latencies are short enough for many network hops before users take notice. In","geographically replicated systems, high latencies drive eventually consistent","and datacenter-pinned solutions.","","","## Common distributed systems","","### Outsourced heaps","","- Redis, memcached, ...","- Data fits in memory, complex data structures","- Useful when your language's built-in data structures are slow/awful","- Excellent as a cache","- Or as a quick-and-dirty scratchpad for shared state between platforms","- Not particularly safe","","### KV stores","","- Riak, Couch, Mongo, Cassandra, RethinkDB, HDFS, ...","- Often 1,2,3 dimensions of keys","- O(1) access, sometimes O(range) range scans by ID","- No strong relationships between values","- Objects may be opaque or structured","- Large data sets","- Often linear scalability","- Often no transactions","- Range of consistency models--often optional linearizable/sequential ops.","","### SQL databases","","- Postgres, MySQL, Percona XtraDB, Oracle, MSSQL, VoltDB, CockroachDB, ...","- Defined by relational algebra: restrictions of products of records, etc","- Moderate sized data sets","- Almost always include multi-record transactions","- Relations and transactions require coordination, which reduces scalability","- Many systems are primary-secondary failover","- Access cost varies depending on indexes","- Typically strong consistency (SI, serializable, strict serializable)","","### Search","","- Elasticsearch, SolrCloud, ...","- Documents referenced by indices","- Moderate-to-large data sets","- Usually O(1) document access, log-ish search","- Good scalability","- Typically weak consistency","","### Coordination services","","- Zookeeper, etcd, Consul, ...","- Typically strong (sequential or linearizable) consistency","- Small data sets","- Useful as a coordination primitive for stateless services","","### Streaming systems","","- Storm, Spark...","- Usually custom-designed, or toolkits to build your own.","- Typically small in-memory data volume","- Low latencies","- High throughput","- Weak consistency","","### Distributed queues","","- Kafka, Kestrel, Rabbit, IronMQ, ActiveMQ, HornetQ, Beanstalk, SQS, Celery, ...","- Journals work to disk on multiple nodes for redundancy","- Useful when you need to acknowledge work now, and actually do it later","- Send data reliably between stateless services","- The *only* one I know that won't lose data in a partition is Kafka"," - Maybe SQS?","- Queues do not improve end-to-end latency"," - Always faster to do the work immediately","- Queues do not improve mean throughput"," - Mean throughput limited by consumers","- Queues do not provide total event ordering when consumers are concurrent"," - Your consumers are almost definitely concurrent","- Likewise, queues don't guarantee event order with async consumers"," - Because consumer side effects could take place out of order"," - So, don't rely on order","- Queues can offer at-most-once or at-least-once delivery"," - Anyone claiming otherwise is trying to sell you something"," - Recovering exactly-once delivery requires careful control of side effects"," - Make your queued operations idempotent","- Queues do improve burst throughput"," - Smooth out load spikes","- Distributed queues also improve fault tolerance (if they don't lose data)"," - If you don't need the fault-tolerance or large buffering, just use TCP"," - Lots of people use a queue with six disk writes and fifteen network hops"," where a single socket write() could have sufficed","- Queues can get you out of a bind when you've chosen a poor runtime","","### Review","","We use data structure stores as outsourced heaps: they're the duct tape of","distributed systems. KV stores and relational databases are commonly deployed","as systems of record; KV stores use independent keys and are not well-suited to","relational data, but offer improved scalability and partial failure vs SQL","stores, which offer rich queries and strong transactional guarantees.","Distributed search and coordination services round out our basic toolkit for","building applications. Streaming systems are applied for continuous,","low-latency processing of datasets, and tend to look more like frameworks than","databases. Their dual, distributed queues, focus on the *messages* rather","than the *transformations*.","","","## A Pattern Language","","- General recommendations for building distributed systems"," - Hard-won experience"," - Repeating what other experts tell me"," - Over beers"," - Hearsay"," - Oversimplifications"," - Cargo-culting"," - Stuff I just made up"," - YMMV","","### Don't distribute","","- Rule 1: don't distribute where you don't have to"," - Local systems have reliable primitives. Locks. Threads. Queues. Txns."," - When you move to a distributed system, you have to build from ground up."," - Is this thing small enough to fit on one node?"," - \"I have a big data problem\""," - Softlayer will rent you a box with 3TB of ram for $5000/mo."," - Supermicro will sell a 6TB box for ~$115,000 total."," - Modern computers are FAST."," - Production JVM HTTP services I've known have pushed 50K requests/sec"," - Parsing JSON events, journaling to disk, pushing to S3"," - Protocol buffers over TCP: 10 million events/sec"," - 10-100 event batches/message, in-memory processing"," - Can this service tolerate a single node's guarantees?"," - Could we just stand up another one if it breaks?"," - Could manual intervention take the place of the distributed algorithm?","","### Use an existing distributed system","","- If we have to distribute, can we push the work onto some other software?"," - What about a distributed database or log?"," - Can we pay Amazon to do this for us?"," - Conversely, what are the care and feeding costs?"," - How much do you have to learn to use/operate that distributed system?","","### Never fail","","- Buy really expensive hardware","- Make changes to software and hardware in a controlled fashion"," - Dry-run deployments against staging environments","- Possible to build very reliable networks and machines"," - At the cost of moving slower, buying more expensive HW, finding talent"," - HW/network failure still *happens*, but sufficiently rare =\u003e low priority","","### Accept failure","","- Distributed systems aren't just characterized by *latency*, but by"," *recurrent, partial failure*","- Can we accept this failure and move on with our lives?"," - What's our SLA anyway?"," - Can we recover by hand?"," - Can we pay someone to fix it?"," - Could insurance cover the damage?"," - Could we just call the customer and apologize?","- Sounds silly, but may be much cheaper"," - We can never prevent 100% of system failures"," - Consciously choosing to recover *above* the level of the system"," - This is how financial companies and retailers do it!","","### Recovery First","","- Assume a failure has just occurred: how will you recover?","- Make this recovery the *default* path of execution","- Writing recovery-first code keeps you from punting on error handling","- Exercising recovery code by default means you know it works","- Recovery by default means you don't have to worry about different semantics"," during a real fault","- If necessary, introduce a happy path for performance optimization"," - But you lose some of these advantages!","","### Reconciliation Loops","","- You've got a complex, stateful system, and want to move it somewhere","- Could devise a plan of changes, and apply those changes in order"," - But what if some change breaks? How do you recover?","- Instead, maintain a *target*: a representation of what you *want* the system"," to become","- Next, write a function that looks at the current state, and *diffs* it with"," the target","- Use that diff to find a step that moves the system closer to the target","- Repeat indefinitely","- Robust to faults and interference"," - What if your admin is tweaking things by hand?"," - What if two instances of the control system are running concurrently?","- Deployed to great effect in systems like [Borg \u0026 Kubernetes](https://queue.acm.org/detail.cfm?id=2898444)","- Also applicable to keeping data in sync between systems"," - Making sure every order is shipped \u0026 billed, for instance","- Again, we're looking for *monotonicity*","","### Backups","","- Backups are essentially sequential consistency, BUT you lose a window of ops."," - When done correctly"," - Some backup programs don't snapshot state, which leads to FS or DB"," corruption"," - Broken fkey relationships, missing files, etc..."," - Allow you to recover in a matter of minutes to days"," - But more than fault recovery, they allow you to step back in time"," - Useful for recovering from logical faults"," - Distributed DB did its job correctly, but you told it to delete key"," data","","### Redundancy","","- OK, so failure is less of an option","- Want to *reduce the probability of failure*","- Have the same state and same computation take place on several nodes"," - I'm not a huge believer in active-spare"," - Spare might have cold caches, broken disks, old versions, etc"," - Spares tend to fail when becoming active"," - Active-active wherever possible"," - Predictability over efficiency"," - Also not a huge fan of only having 2 copies"," - Node failure probabilities just too high"," - OK for not-important data"," - I generally want three copies of data"," - For important stuff, 4 or 5"," - For Paxos and other majority-quorum systems, odd numbers: 3, 5, 7"," common"," - Common DR strategy: Paxos across 5 nodes; 3 or 4 in primary DC"," - Ops can complete as soon as the local nodes ack; low latencies"," - Resilient to single-node failure (though latencies will spike)"," - But you still have a sequentially consistent backup in the other DC"," - So in the event you lose an entire DC, all's not lost"," - See Camille Fournier's talks on ZK deployment","- Redundancy improves availability so long as failures are uncorrelated"," - Failures are not uncorrelated"," - Disks from the same batch failing at the same time"," - Same-rack nodes failing when the top-of-rack switch blows"," - Same-DC nodes failing when the UPS blows"," - See entire EC2 AZ failures"," - Running the same bad computation on every node will break every node"," - Expensive queries"," - Riak list-keys"," - Cassandra doomstones"," - Cascading failures"," - Thundering-herd"," - TCP incast","","### Sharding","","- The problem is too big","- Break the problem into parts small enough to fit on a node"," - Not too small: small parts =\u003e high overhead"," - Not too big: need to rebalance work units gradually from node to node"," - Somewhere around 10-100 work units/node is ideal, IMO","- Ideal: work units of equal size"," - Beware hotspots"," - Beware changing workloads with time","- Know your bounds in advance"," - How big can a single part get before overwhelming a node?"," - How do we enforce that limit *before* it sinks a node in prod?"," - Then sinks all the other nodes, one by one, as the system rebalances","- Allocating shards to nodes"," - Often built in to DB"," - Good candidate for ZK, Etcd, and so on"," - See Boundary's Ordasity","","### Independent domains","","- Sharding is a specific case of a more general pattern: avoiding coordination"," - Keep as much independent as possible"," - Improves fault tolerance"," - Improves performance"," - Reduces complexity"," - Sharding for scalability"," - Avoiding coordination via CRDTs"," - Flake IDs: *mostly* time-ordered identifiers, zero-coordination"," - See http://yellerapp.com/posts/2015-02-09-flake-ids.html"," - Partial availability: users can still use some parts of the system"," - Processing a queue: more consumers reduces the impact of expensive events","","### ID structure","","- Things in our world have to have unique identifiers"," - At scale, ID structure can make or break you"," - Consider your access patterns"," - Scans"," - Sorts"," - Shards"," - Sequential IDs require coordination: can you avoid them?"," - Flake IDs, UUIDs, ..."," - For *shardability*, can your ID map directly to a shard?"," - SaaS app: object ID can also encode customer ID"," - Twitter: tweet ID can encode user ID","","### Immutable values","","- Data that never changes is trivial to store"," - Never requires coordination"," - Cheap replication and recovery"," - Minimal repacking on disk","- Useful for Cassandra, Riak, any LSM-tree DB."," - Or for logs like Kafka!","- Easy to reason about: either present or it's not"," - Eliminates all kinds of transactional headaches"," - Extremely cachable","- Extremely high availability and durability, tunable write latency"," - Low read latencies: can respond from closest replica"," - Especially valuable for geographic distribution","- Requires garbage collection!"," - But there are good ways to do this","","### Mutable identities","","- Pointers to immutable values","- Pointers are small! Only metadata!"," - Can fit huge numbers of pointers on a small DB"," - Good candidate for consensus services or relational DBs","- And typically, not many pointers in the system"," - Your entire DB could be represented by a single pointer"," - Datomic only has ~5 identities","- Strongly consistent operations over identities can be *backed* by immutable"," HA storage"," - Take advantage of AP storage latencies and scale"," - Take advantage of strong consistency over small datasets provided by"," consensus systems"," - Write availability limited by identity store"," - But, reads eminently cachable if you only need sequential consistency"," - Can be even cheaper if you only need serializability"," - See Rich Hickey's talks on Datomic architecture"," - See Pat Helland's 2013 RICON West keynote on Salesforce's storage","","### Confluence","","- Systems which are order-independent are easier to construct and reason about","- Also helps us avoid coordination","- CRDTs are confluent, which means we can apply updates without waiting","- Immutable values are trivially confluent: once present, fixed","- Streaming systems can leverage confluence as well:"," - Buffer events, and compute+flush when you know you've seen everything"," - Emit partial results so you can take action now, e.g. for monitoring"," - When full data is available, merge with + or max"," - Bank ledgers are (mostly) confluent: txn order doesn't affect balance"," - But when you need to enforce a minimum balance, no longer confluent"," - Combine with a sealing event (e.g. the day's end) to recover confluence","- See Aiken, Widom, \u0026 Hellerstein 1992, \"Behavior of Database Production Rules\"","","### Backpressure","","- Services which talk to each other are usually connected by *queues*","- Service and queue capacity is finite","- How do you handle it when a downstream service is unable to handle load?"," 1. Consume resources and explode"," 2. Shed load. Start dropping requests."," 3. Reject requests. Ignore the work and tell clients it failed."," 4. Apply backpressure to clients, asking them to slow down.","- 2-4 allow the system to catch up and recover"," - But backpressure reduces the volume of work that has to be retried","- Backpressure defers choice to producers: compositional"," - Clients of load-shedding systems are locked into load-shedding"," - They have no way to tell that the system is hosed"," - Clients of backpressure systems can apply backpressure to *their clients*"," - Or shed load, if they choose"," - If you're making an asynchronous system, *always* include backpressure"," - Your users will thank you later","- Fundamentally: *bounding resources*"," - Request timeouts (bounded time)"," - Exponential backoffs (bounded use)"," - Bounded queues"," - Bounded concurrency","- See Zach Tellman, \"Everything Will Flow\"","","### Services for domain models","","- The problem is composed of interacting logical pieces","- Pieces have distinct code, performance, storage needs"," - Monolithic applications are essentially *multitenant* systems"," - Multitenancy is tough"," - But its often okay to run multiple logical \"services\" in the same process","- Divide your system into logical services for discrete parts of the domain"," model"," - OO approach: each *noun* is a service"," - User service"," - Video service"," - Index service"," - Functional approach: each *verb* is a service"," - Auth service"," - Search service"," - Dispatch/routing service"," - Most big systems I know of use a hybrid"," - Services for nouns is a good way to enforce *datatype invariants*"," - Services for verbs is a good way to enforce *transformation invariants*"," - So have a basic User service, which is used *by* an Auth service"," - Where you draw the line... well that's tricky"," - Services come with overhead: have as few as possible"," - Consider work units"," - Separate services which need to scale independently"," - Colocate services with tight dependencies and tight latency budgets"," - Colocate services which use complementary resources (e.g. disk and CPU)"," - By hand: Run memcache on rendering nodes"," - Newer shops: Google Borg, Mesos, Kubernetes"," - Services should encapsulate and abstract"," - Try to build trees instead of webs"," - Avoid having outsiders manipulate a service's data store directly","","### Structure Follows Social Spaces","","- Production software is a fundamentally social artifact","- Natural alignment: a team or person owns a specific service"," - Jo Freeman, \"The Tyranny of Structurelessness\""," - https://www.jofreeman.com/joreen/tyranny.htm"," - Responsibility and power should be explicit"," - Rotate people through roles to prevent fiefdoms"," - People own *problem spaces*, not services"," - Promotes information sharing"," - But don't rotate too often"," - Ramp-up costs in software are very high"," - Pair Freeman with Downs' \"Inside Bureaucracy\""," - https://www.rand.org/content/dam/rand/pubs/papers/2008/P2963.pdf","- As the team grows, its mission and thinking will formalize"," - So too will services and their boundaries"," - Gradually accruing body of assumptions about service relation to the world"," - Punctuated by rewrites to respond to changing external pressures"," - Tushman \u0026 Romanelli, 1985: Organizational Transformation as Punctuated Equilibrium","- Services can be libraries"," - Initially, *all* your services should be libraries"," - Perfectly OK to depend on a user library in multiple services"," - Libraries with well-defined boundaries are easy to extract into"," services later","- Social structure governs the library/service boundary"," - With few users of a library, or tightly-coordinated users, changes are easy"," - But across many teams, users have varying priorities and must be convinced"," - Why should users do work to upgrade to a new library version?"," - Services *force* coordination through a defined API deprecation lifecycle"," - You can also enforce this with libraries through code review \u0026 tooling","- Services enable centralized control"," - Your performance improvements affect everyone instantly"," - Gradually shift to a new on-disk format or backing database"," - Instrument use of the service in one place"," - Harder to do these things with libraries","- Services have costs"," - The failure-complexity and latency overhead of a network call"," - Tangled food web of service dependencies"," - Hard to statically analyze codepaths"," - You thought library API versioning was hard"," - Additional instrumentation/deployment","- Services can use good client libraries"," - That library might be \"Open a socket\" or an HTTP client"," - Leverage HTTP headers!"," - Accept headers for versioning"," - Lots of support for caching and proxying"," - Haproxy is an excellent router for both HTTP and TCP services"," - Eventually, library might include mock IO"," - Service team is responsible for testing that the service provides an API"," - When the API is known to be stable, every client can *assume* it works"," - Removes the need for network calls in test suites"," - Dramatic reduction in test runtime and dev environment complexity","","### Cross-service coordination","","- Coordination between services requires special protocols"," - Have to re-invent transactions"," - Go commutative where possible"," - Sagas"," - Was written for a single-node world: we have to be clever in distributed contexts"," - Transactions must be idempotent, OR commute with rollbacks"," - [Typhon/Cerberus](http://www.cs.ucsb.edu/~vaibhavarora/Typhon-Ieee-Cloud-2017.pdf)"," - Protocols for causal consistency over multiple data stores"," - e.g. If Lupita blocks Miss Angela, then posts, Miss Angela can't see it"," - Typhon: A single logical *entity* has *data item* representations in"," different data stores"," - Assumes datastores are serializable or offer atomic read/cas on items"," - Transactions which access same entity AND T1 happens-before T2 get a"," causal dependency edge T1 -\u003e T2"," - Cerberus: protocol for transactions involving a single entity x"," - Writes can only affect one representation of x"," - Any number of reads across representations of x"," - Global metadata: a version vector for each entity (GVV)"," - Each representation metadata:"," - Update version vector (UVV): versions known as of last update"," - Read version vector (RVV): versions known as of last read"," - Conflicts detected when GVV \u003c UVV/RVV"," - Two phases:"," - Reads"," - Check GVV for entity x"," - Perform reads of x on each (asked-for) representation"," - At each representation: check RVV \u003c= GVV, update RVV"," - Writes"," - Send write to representation"," - Check UVV \u003c= GVV \u0026 RVV \u003c= GVV"," - Commit"," - Update representation and RVV/UVV ensuring RVV/UVV unchanged"," - New UVV constructed by incrementing i'th entry of existing UVV"," - General-purpose transactions"," - [Calvin](http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf)"," - Serializable (or strict-1SR) transactions"," - Deterministic txns enqueued into sharded global log"," - Log ensures txn order"," - Application at replicas/shards requires no further coordination"," - Minimum latency floor for log windows"," - [CockroachDB](https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/)"," - Serializable"," - Assumes linearizable stores"," - Assumes semi-sync clocks"," - Similar to a more tractable Spanner","","### Migrations","","- Migrations are hard."," - There's no silver bullet"," - But some techniques can make your life easier","- Hard cut"," - Write new system and migration to copy old data to it"," - Have dependent services talk to both--but in practice, only one."," - Turn off old system"," - Copy data"," - Start up new system"," - Tradeoffs!"," - Don't have to worry about in-flight data"," - Simple migration scripts: just read all data and write to new datastore"," - Requires downtime proportional to migration script"," - Can sometimes scope this to a single shard/user/domain at a time","- Incremental"," - Write new system B"," - Deploy alongside original A"," - Dependent services talk to both"," - Ideal: find all readers, have every reader talk to both A and B"," - Then start writing to B"," - This frees you from having to worry about readers who only know about A"," - Consistency nightmares; need to trace all data dependencies"," - Tradeoffs!"," - Reduced/no downtime"," - But complex reasoning about data dependencies required","- Wrapper services"," - Operationally speaking, it can be tricky to FIND and change all users of A"," - So... don't. Introduce a wrapper service W which proxies to A"," - Introduce B, and make changes so that W talks to B as well"," - When A is phased out, remove W and talk directly to B."," - Allows for centralized metrics, errors, comparison of behavior, etc","- Eventual atomicity"," - Imagine you write each update to old service A, then new service B"," - At some point, your write to A will succeed, and B will fail. What then?"," - Can use read-repair: read A and B, fill in missing updates"," - But this requires mergeability: only works for things like CRDTs"," - Can use a reconciliation process"," - Iterate over whole DB, look for changes, apply to both."," - Also requires something like a CRDT"," - Can use a Saga"," - All updates go to durable queue"," - Queue worker retries until updates applied to A and B"," - Might require ordering updates to avoid state divergence"," - Potentially *global* serialization"," - Be aware of the DB's consistency model","- Isolation"," - Imagine Jane writes w1 to A, then B"," - Concurrently, Naomi writes w2 to B, then A"," - Result: A = w2, B = w1"," - Eventual atomicity isn't enough to prevent divergence"," - Can reduce issues by picking a standard order: always A then B (or B then A)"," - But imagine"," - Jane writes A = w1"," - Naomi writes A = w2"," - Naomi writes B = w2"," - Jane writes B = w1"," - We've got a mixed result again. Shoot."," - Can mitigate using CRDTs"," - Or, if A and B are sequentially consistent, can use a CaS operation to"," ensure agreement on order"," - \"Write w2 iff the last write was w1\""," - If operations affect multiple keys, the CaS logic has to be applied at that"," level too","- Helpful properties for incremental migrations"," - Determinism"," - Avoid having the DB generate random numbers, automatic IDs, timestamps"," - Easier to apply updates to two datastores and get the same results"," - Idempotence"," - Lets you retry updates freely"," - Commutativity"," - Removes the need for serializing updates"," - CRDTs: associativity, commutativity, idempotence."," - Immutability: trivial CRDTs"," - Statelessness: no state to worry about!"," - Just make sure you talk to external stateful stuff the same way","- What about swapping out queues?"," - As we've touched, queue systems should already be designed for idempotency,"," and ideally commutativity"," - If so, this is (relatively) easy"," - Workers consume from both queues"," - Flip producers to send messages only to the new queue"," - Wait for old queue to be exhausted"," - Decommission old queue"," - But we WANTED order???"," - You're going to need to reconstruct it"," - One option: single producer tightly coupled to queue"," - Writes each message m to both A and B; doesn't move on until both ack"," - This enforces that both A and B agree on order"," - Consumers can treat A and B identically: consume just from A or B"," - Again, assumes idempotence!"," - Another option: sequence numbers, order reconstructed at client"," - e.g. assign sequence numbers to each value"," - Use the queue offsets from A?"," - Use a consensus system?"," - Clients read sequence numbers, store in internal buffer, apply in order","","","### Review","","When possible, try to use a single node instead of a distributed system. Accept","that some failures are unavoidable: SLAs and apologies can be cost-effective.","To handle catastrophic failure, we use backups. To improve reliability, we","introduce redundancy. To scale to large problems, we divide the problem into","shards. Immutable values are easy to store and cache, and can be referenced by","mutable identities, allowing us to build strongly consistent systems at large","scale. As software grows, different components must scale independently,","and we break out libraries into distinct services. Service structure goes","hand-in-hand with teams.","","","","## Production Concerns","","- More than design considerations","- Proofs are important, but real systems do IO","","### Distributed systems are supported by your culture","","- Understanding a distributed system in production requires close cooperation"," of people with many roles"," - Development"," - QA"," - Operations","- Empathy matters"," - Developers have to care about production"," - Ops has to care about implementation"," - Good communication enables faster diagnosis","","### Test everything","","- Type systems are great for preventing logical errors"," - Which reduces your testing burden","- However, they are *not* great at predicting or controlling runtime"," performance","- So, you need a solid test suite"," - Ideally, you want a *slider* for rigorousness"," - Quick example-based tests that run in a few seconds"," - More thorough property-based tests that can run overnight"," - Be able to simulate an entire cluster in-process"," - Control concurrent interleavings with simulated networks"," - Automated hardware faults","- Testing distributed systems is much, much harder than testing local ones"," - Huge swath of failure modes you've never even heard of"," - Combinatorial state spaces"," - Bugs can manifest only for small/large/intermediate time/space/concurrency","","### \"It's Slow\"","","- Jeff Hodges: The worst bug you'll ever hear is \"it's slow\""," - Happens all the time, really difficult to localize"," - Because the system is distributed, have to profile multiple nodes"," - Not many profilers are built for this"," - Sigelman et al, 2010: Dapper, a Large-Scale Distributed Systems Tracing"," Infrastructure"," - Zipkin"," - Big tooling investment"," - Profilers are good at finding CPU problems"," - But high latency is often a sign of IO, not CPU"," - Disk latency"," - Network latency"," - GC latency"," - Queue latency"," - Try to localize problem using application-level metrics"," - Then dig in to process and OS performance"," - Latency variance between nodes doing the same work is an important signal"," - 1/3 nodes slow: likely node HW, re-route"," - 3/3 nodes slow: likely a logical fault: look at shard size, workload, queries"," - Tail latencies are magnified by fanout workloads"," - [Jeff Dean, 2013: The Tail at Scale](https://research.google.com/pubs/pub40801.html)"," - Consider speculative parallelism","","### Instrument everything","","- Slowness (and outright errors) in prod stem from the interactions *between*"," systems"," - Why? Because your thorough test suite probably verified that the single"," system was mostly correct"," - So we need a way to understand what the system is doing in prod"," - In relation to its dependencies"," - Which can, in turn, drive new tests"," - In a way, good monitoring is like continuous testing"," - But not a replacement: these are distinct domains"," - Both provide assurance that your changes are OK"," - Want high-frequency monitoring"," - Production behaviors can take place on 1ms scales"," - TCP incast"," - ~1ms resolution *ideally*"," - Ops response time, in the limit, scales linearly with observation latency"," - ~1 second end to end latency"," - Ideally, millisecond latencies, maybe ms resolution too"," - Usually cost-prohibitive; back off to 1s or 10s"," - Sometimes you can tolerate 60s"," - And for capacity planning, hourly/daily seasonality is more useful"," - Instrumentation should be tightly coupled to the app"," - Measure only what matters"," - Responding to requests is important"," - Node CPU doesn't matter as much"," - Key metrics for most systems"," - Apdex: successful response WITHIN latency SLA"," - Latency profiles: 0, 0.5, 0.95, 0.99, 1"," - Percentiles, not means"," - BTW you can't take the mean of percentiles either"," - Overall throughput"," - Queue statistics"," - Queries per query"," - Subjective experience of other systems latency/throughput"," - The DB might think it's healthy, but clients could see it as slow"," - Combinatorial explosion--best to use this when drilling into a failure"," - You probably have to write this instrumentation yourself"," - Invest in a metrics library"," - Out-of-the-box monitoring usually doesn't measure what really matters: your"," app's behavior"," - But it can be really useful in tracking down causes of problems"," - Host metrics like CPU, disk, etc"," - Where your app does something common (e.g. rails apps) tools like New"," Relic work well"," - Shard metrics by client"," - Helpful when users have varying workloads"," - Can tune thresholds to be appropriate for that client"," - A few for major clients, another bucket for \"the rest\""," - Superpower: distributed tracing infra (Zipkin, Dapper, etc)"," - Significant time investment"," - [Mystery Machine](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-chow.pdf)"," - Automatic inference of causal relationships between services from trace data"," - Identification of critical paths"," - Performance modeling new algorithms before implementation","","### Logging","","- Logging is less useful at scale"," - Problems may not be localized to one node"," - As requests touch more services, must trace through many logfiles"," - Invest in log collection infrastructure"," - ELK, Splunk, etc"," - Unstructured information is harder to aggregate"," - Log structured events","","### Shadow traffic","","- Load tests are only useful insofar as the simulated load matches the actual"," load","- Consider dumping production traffic"," - Awesome: kill a process with SIGUSR1, it dumps five minutes of request load"," - Awesome: tcpdump/tcpreplay harnesses for requests"," - Awesome: shadowing live prod traffic to your staging/QA nodes","- See Envoy from Lyft","","### Versioning","","- Protocol versioning is, as far as I know, a wide-open problem"," - Do include a version tag with all messages"," - Do include compatibility logic"," - Inform clients when their request can't be honored"," - And instrument this so you know which systems have to be upgraded","","### Rollouts","","- Rollouts are often how you fix problems","- Spend the time to get automated, reliable deploys"," - Amplifies everything else you do"," - Have nodes smoothly cycle through to prevent traffic interruption"," - This implies you'll have multiple versions of your software running at"," once"," - Versioning rears its ugly head"," - Inform load balancer that they're going out of rotation"," - Coordinate to prevent cascading failures","- Roll out only to a fraction of load or fraction of users"," - Gradually ramp up number of users on the new software"," - Either revert or roll forward when you see errors"," - Consider shadowing traffic in prod and comparing old/new versions"," - Good way to determine if new code is faster \u0026 correct","","### Automated Control","","- Automated failure handling is good","- But not too much"," - \"Ironies of Automation\", Bainbridge 1983"," - https://pdfs.semanticscholar.org/0713/bb9d9b138e4e0a15406006de9b0cddf68e28.pdf"," - Controlling complex systems requires operators with accurate mental models"," of how the system is behaving, ought to behave, and how it responds to"," control inputs"," - If you aren't engaged in the control process, you forget these things"," - Human takeover is then challenging!"," - Humans literally cannot pay attention to things that don't change for more"," than ~30 minutes"," - Merely monitoring, as opposed to running, a complex process \"deskills\""," operators"," - So... consider *deliberately* bringing humans into the control process"," - Catastrophic failure is easy to identify"," - But automated control tends to mask failure: prefailure trends are not"," apparent until well outside the automated-control regime"," - If humans are expected to verify automated control decisions, humans must"," be *capable* of verifying that reasoning"," - Huge ML models (for example) make inscrutable decisions","","### Feature flags","","- We want incremental rollouts of a changeset after a deploy"," - Introduce features one by one to watch their impact on metrics"," - Gradually shift load from one database to another"," - Disable features when rollout goes wrong","- We want to obtain partial availability when some services are degraded"," - Disable expensive features to speed recovery during a failure","- Use a highly available coordination service to decide which codepaths to"," enable, or how often to take them"," - This service should have minimal dependencies"," - Don't use the primary DB","- When things go wrong, you can *tune* the system's behavior"," - When coordination service is down, fail *safe*!","","### Chaos engineering","","- Breaking things in production"," - Forces engineers to handle failure appropriately *now*, not in response to"," an incident later"," - Identifies unexpected dependencies in the critical path"," - \"When the new stats service goes down, it takes the API with it. Are you"," *sure* that's necessary?\""," - Requires good instrumentation and alerting, so you can measure impact of"," events"," - Limited blast radius"," - Don't nuke an entire datacenter every five minutes"," - But *do* try it once a quarter"," - Don't break *too* many nodes in a replication group"," - Break only a small fraction of requests/users at a time","","### Oh no, queues","","- Every queue is a place for things to go horribly, horribly wrong"," - No node has unbounded memory. Your queues *must* be bounded"," - But how big? Nobody knows"," - Instrument your queues in prod to find out","- Little's Law: mean queue depth = mean arrival rate * mean latency"," - This is distribution-independent!","- Queues exist to smooth out fluctuations in load"," - Improves throughput at expense of latency"," - If your load is higher than capacity, no queue will save you"," - https://ferd.ca/queues-don-t-fix-overload.html"," - Shed load or apply backpressure when queues become full"," - Instrument this"," - When load-shedding occurs, alarm bells should ring"," - Backpressure is visible as upstream latency"," - Instrument queue depths"," - High depths is a clue that you need to add node capacity"," - End to end queue latency should be smaller than fluctuation timescales"," - Raising the queue size can be tempting, but is a vicious cycle"," - All of this is HARD. I don't have good answers for you"," - Ask Jeff Hodges why it's hard: see his RICON West 2013 talk"," - See Zach Tellman - Everything Will Flow","","","## Review","","Running distributed systems requires cooperation between developers, QA, and","operations engineers. Static analysis, and a test suite including example- and","property-based tests, can help ensure program correctness, but understanding","production behavior requires comprehensive instrumentation and alerting. Mature","distributed systems teams often invest in tooling: traffic shadowing,","versioning, incremental deploys, and feature flags. Finally, queues require","special care.","","## Further reading","","### Online","","- Mixu has a delightful book on distributed systems with incredible detail. http://book.mixu.net/distsys/","- Jeff Hodges has some excellent, production-focused advice. https://www.somethingsimilar.com/2013/01/14/notes-on-distributed-systems-for-young-bloods/ https://player.vimeo.com/video/42898664","- The Fallacies of Distributed Computing is a classic text on mistaken assumptions we make designing distributed systems. http://www.rgoarchitects.com/Files/fallacies.pdf","- Christopher Meiklejohn has a list of key papers in distributed systems. http://christophermeiklejohn.com/distributed/systems/2013/07/12/readings-in-distributed-systems.html","- Dan Creswell has a lovely reading list. https://dancres.github.io/Pages/","","### Trees","","- Martin Kleppmann's [Designing Data-Intensive"," Applications](https://dataintensive.net/) offers a thorough tour of"," distributed systems for practitioners.","- Nancy Lynch's \"Distributed Algorithms\" is a comprehensive overview of the"," field from a more theoretical perspective","- Suggestions from students:"," - Donella Meadows' \"Thinking in Systems\""," - \"Database Internals: A Deep Dive into How Distributed Data Systems Work\""," - Perhaps more intense than Kleppman"," - \"Thinking in Systems\""],"stylingDirectives":[[[0,40,"pl-mh"],[2,40,"pl-en"]],[],[],[],[[41,42,"pl-s"]],[[12,13,"pl-s"],[13,14,"pl-s"],[14,45,"pl-corl"],[45,46,"pl-s"]],[],[],[[0,1,"pl-s"],[8,9,"pl-s"],[9,10,"pl-s"],[10,48,"pl-corl"],[48,49,"pl-s"]],[],[],[],[[0,8,"pl-mh"],[3,8,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[11,26,"pl-corl"]],[[2,3,"pl-v"],[18,56,"pl-corl"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"],[13,14,"pl-s"],[19,20,"pl-s"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[15,16,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[13,14,"pl-s"],[19,20,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[32,33,"pl-s"],[46,47,"pl-s"],[47,48,"pl-s"],[48,126,"pl-corl"],[126,127,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[21,22,"pl-s"],[26,27,"pl-s"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[0,34,"pl-mh"],[3,34,"pl-en"]],[],[],[],[[0,65,"pl-ent"],[0,2,"pl-ent"]],[[0,60,"pl-ent"],[0,2,"pl-ent"]],[[0,12,"pl-ent"],[0,2,"pl-ent"]],[[0,0,"pl-ent"]],[[0,1,"pl-v"],[16,18,"pl-c1"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,21,"pl-mh"],[3,21,"pl-en"]],[],[[0,1,"pl-v"],[46,47,"pl-s"],[51,52,"pl-s"]],[[2,3,"pl-v"],[18,19,"pl-s"],[28,29,"pl-s"],[31,32,"pl-s"],[38,39,"pl-s"],[44,45,"pl-s"],[51,52,"pl-s"]],[],[[0,9,"pl-mh"],[4,9,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[15,16,"pl-s"],[22,23,"pl-s"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[4,5,"pl-s"],[16,17,"pl-s"],[17,18,"pl-s"],[18,59,"pl-corl"],[59,60,"pl-s"]],[[2,3,"pl-v"],[4,5,"pl-s"],[39,40,"pl-s"],[40,41,"pl-s"],[41,90,"pl-corl"],[90,91,"pl-s"]],[[2,3,"pl-v"],[4,5,"pl-s"],[16,17,"pl-s"],[17,18,"pl-s"],[18,83,"pl-corl"],[83,84,"pl-s"]],[[2,3,"pl-v"],[4,5,"pl-s"],[21,22,"pl-s"],[22,23,"pl-s"],[23,102,"pl-corl"],[102,103,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,29,"pl-mh"],[4,29,"pl-en"]],[],[[0,1,"pl-v"],[23,24,"pl-s"],[31,32,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[42,43,"pl-s"],[51,52,"pl-s"]],[[0,1,"pl-v"],[16,17,"pl-s"],[21,22,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[30,31,"pl-s"],[41,42,"pl-s"]],[],[[0,24,"pl-mh"],[4,24,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,29,"pl-mh"],[4,29,"pl-en"]],[],[[0,1,"pl-v"],[63,64,"pl-s"],[68,69,"pl-s"]],[],[[0,25,"pl-mh"],[4,25,"pl-en"]],[],[[0,1,"pl-v"],[60,61,"pl-s"],[65,66,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[35,36,"pl-s"],[46,47,"pl-s"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[20,21,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[0,25,"pl-mh"],[3,25,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[55,56,"pl-s"],[67,68,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[6,86,"pl-corl"]],[[4,5,"pl-v"],[6,125,"pl-corl"]],[],[[0,22,"pl-mh"],[3,22,"pl-en"]],[],[[0,7,"pl-mh"],[4,7,"pl-en"]],[],[[0,1,"pl-v"],[6,7,"pl-s"],[12,13,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[12,13,"pl-s"],[24,25,"pl-s"]],[],[],[[0,7,"pl-mh"],[4,7,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[38,39,"pl-s"],[43,44,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[24,25,"pl-s"],[29,30,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[],[[0,9,"pl-mh"],[3,9,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-s"],[8,9,"pl-s"]],[[0,1,"pl-v"]],[],[[0,15,"pl-mh"],[4,15,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[8,165,"pl-corl"]],[[2,3,"pl-v"],[15,16,"pl-s"],[25,26,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[6,50,"pl-corl"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,59,"pl-corl"]],[[2,3,"pl-v"],[43,44,"pl-s"],[54,55,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[6,86,"pl-corl"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[6,70,"pl-corl"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[11,119,"pl-corl"]],[[2,3,"pl-v"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[57,58,"pl-s"],[58,68,"pl-c1"],[68,69,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[4,5,"pl-s"],[5,27,"pl-c1"],[27,28,"pl-s"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[],[[0,17,"pl-mh"],[4,17,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"],[2,3,"pl-s"],[3,27,"pl-c1"],[27,28,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[63,64,"pl-s"],[68,69,"pl-s"]],[[2,3,"pl-v"]],[],[[0,23,"pl-mh"],[4,23,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,53,"pl-corl"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[],[],[],[[0,9,"pl-mh"],[3,9,"pl-en"]],[],[],[],[],[],[[24,25,"pl-s"],[35,36,"pl-s"]],[],[],[],[],[[0,15,"pl-mh"],[3,15,"pl-en"]],[],[[0,1,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,23,"pl-mh"],[4,23,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"],[28,29,"pl-s"],[36,37,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"],[21,22,"pl-s"],[24,25,"pl-s"]],[],[[0,1,"pl-v"]],[],[[0,28,"pl-mh"],[4,28,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[],[],[[0,14,"pl-mh"],[3,14,"pl-en"]],[],[[0,1,"pl-v"]],[],[[0,19,"pl-mh"],[4,19,"pl-en"]],[],[[0,1,"pl-v"]],[],[[0,20,"pl-mh"],[4,20,"pl-en"]],[],[[0,1,"pl-v"],[66,67,"pl-s"],[72,73,"pl-s"]],[],[],[[0,20,"pl-mh"],[4,20,"pl-en"]],[],[[0,1,"pl-v"]],[],[],[[0,23,"pl-mh"],[4,23,"pl-en"]],[],[[0,1,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[42,43,"pl-s"],[53,54,"pl-s"]],[[0,1,"pl-v"]],[],[[0,1,"pl-v"]],[],[[0,26,"pl-mh"],[4,26,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,19,"pl-mh"],[4,19,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[40,41,"pl-s"],[48,49,"pl-s"]],[],[[0,1,"pl-v"]],[],[[0,24,"pl-mh"],[4,24,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[19,20,"pl-s"],[32,33,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[19,20,"pl-s"],[31,32,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[19,20,"pl-s"],[31,32,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"],[19,20,"pl-s"],[28,29,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[6,7,"pl-v"],[29,30,"pl-s"],[71,72,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[4,5,"pl-v"]],[[18,19,"pl-s"],[32,33,"pl-s"]],[[6,7,"pl-s"],[9,10,"pl-s"]],[[6,7,"pl-v"]],[],[[0,37,"pl-mh"],[4,37,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[6,7,"pl-s"],[16,17,"pl-s"],[32,33,"pl-s"],[39,40,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[4,5,"pl-s"],[39,40,"pl-s"],[40,41,"pl-s"],[41,93,"pl-corl"],[93,94,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"],[4,5,"pl-s"],[44,45,"pl-s"],[45,46,"pl-s"],[46,148,"pl-corl"],[148,149,"pl-s"]],[[4,5,"pl-v"]],[],[[4,5,"pl-v"]],[],[[0,12,"pl-mh"],[3,12,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,32,"pl-mh"],[4,32,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[17,18,"pl-s"],[24,25,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[17,18,"pl-s"],[25,26,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[47,48,"pl-s"],[54,55,"pl-s"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[3,4,"pl-v"]],[[3,4,"pl-v"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[],[],[],[],[],[],[],[[0,36,"pl-mh"],[3,36,"pl-en"]],[],[[0,19,"pl-mh"],[4,19,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[50,51,"pl-s"],[54,55,"pl-s"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[36,37,"pl-s"],[42,43,"pl-s"]],[],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[],[[0,9,"pl-mh"],[4,9,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[56,57,"pl-s"],[61,62,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[20,21,"pl-s"],[25,26,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,8,"pl-mh"],[4,8,"pl-en"]],[],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[],[[0,37,"pl-mh"],[3,37,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[],[],[[0,1,"pl-v"],[31,32,"pl-s"],[42,43,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[55,56,"pl-s"],[59,60,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[33,34,"pl-s"],[46,47,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[21,22,"pl-s"],[28,29,"pl-s"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[36,37,"pl-s"],[40,41,"pl-s"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[],[],[[0,9,"pl-mh"],[4,9,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[54,55,"pl-s"],[56,57,"pl-s"]],[],[],[],[],[],[[26,27,"pl-s"],[28,29,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"],[30,31,"pl-s"],[37,38,"pl-s"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[90,91,"pl-s"],[121,122,"pl-s"],[122,123,"pl-s"],[123,155,"pl-corl"],[155,156,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,7,"pl-mh"],[4,7,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,27,"pl-mh"],[4,27,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,8,"pl-mh"],[4,8,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[54,55,"pl-s"],[59,60,"pl-s"]],[[2,3,"pl-v"],[27,28,"pl-s"],[31,32,"pl-s"]],[[0,1,"pl-v"],[58,59,"pl-s"],[62,63,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,27,"pl-mh"],[3,27,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"],[50,51,"pl-s"],[56,57,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[2,3,"pl-s"],[40,41,"pl-s"],[41,42,"pl-s"],[42,118,"pl-corl"],[118,119,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[16,17,"pl-s"],[17,18,"pl-s"],[18,68,"pl-corl"],[68,69,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[15,16,"pl-s"],[16,17,"pl-s"],[17,76,"pl-corl"],[76,77,"pl-s"]],[[0,1,"pl-v"],[2,3,"pl-s"],[13,14,"pl-s"],[14,15,"pl-s"],[15,93,"pl-corl"],[93,94,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[13,14,"pl-s"],[14,15,"pl-s"],[15,77,"pl-corl"],[77,78,"pl-s"]],[[0,1,"pl-v"],[2,3,"pl-s"],[10,11,"pl-s"],[11,12,"pl-s"],[12,107,"pl-corl"],[107,108,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[20,21,"pl-s"],[21,22,"pl-s"],[22,118,"pl-corl"],[118,119,"pl-s"]],[[2,3,"pl-v"],[8,9,"pl-s"],[20,21,"pl-s"],[21,22,"pl-s"],[22,126,"pl-corl"],[126,127,"pl-s"]],[[0,1,"pl-v"],[2,3,"pl-s"],[9,10,"pl-s"],[10,11,"pl-s"],[11,76,"pl-corl"],[76,77,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[14,15,"pl-s"],[15,16,"pl-s"],[16,34,"pl-corl"],[34,35,"pl-s"]],[],[[0,9,"pl-mh"],[3,9,"pl-en"]],[],[],[],[],[[58,59,"pl-s"],[68,69,"pl-s"]],[],[[0,1,"pl-s"],[7,8,"pl-s"]],[],[],[],[[0,27,"pl-mh"],[3,27,"pl-en"]],[],[[0,1,"pl-v"],[13,14,"pl-s"],[19,20,"pl-s"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[5,6,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[32,33,"pl-s"],[46,47,"pl-s"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[30,31,"pl-s"],[105,106,"pl-s"],[106,107,"pl-s"],[107,170,"pl-corl"],[170,171,"pl-s"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[43,44,"pl-s"],[56,57,"pl-s"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"],[10,11,"pl-s"],[17,18,"pl-s"]],[[10,11,"pl-v"]],[[10,11,"pl-v"]],[[12,13,"pl-v"],[20,21,"pl-s"],[43,44,"pl-s"]],[[14,15,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[19,20,"pl-s"],[30,31,"pl-s"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[0,26,"pl-mh"],[4,26,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[24,25,"pl-s"],[49,50,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[[0,29,"pl-mh"],[3,29,"pl-en"]],[],[[0,20,"pl-mh"],[4,20,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,13,"pl-mh"],[4,13,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,17,"pl-mh"],[4,17,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,25,"pl-mh"],[4,25,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"],[6,7,"pl-s"],[11,12,"pl-s"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[],[],[],[],[],[],[],[],[[56,57,"pl-s"],[65,66,"pl-s"]],[[9,10,"pl-s"],[25,26,"pl-s"]],[],[],[[0,21,"pl-mh"],[3,21,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,20,"pl-mh"],[4,20,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"],[43,44,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,38,"pl-mh"],[4,38,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,14,"pl-mh"],[4,14,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[29,30,"pl-s"],[37,38,"pl-s"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"],[51,52,"pl-s"],[59,60,"pl-s"]],[[2,3,"pl-s"],[29,30,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[36,37,"pl-s"],[42,43,"pl-s"]],[[2,3,"pl-v"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"],[25,26,"pl-s"],[33,34,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,24,"pl-mh"],[4,24,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[22,23,"pl-s"],[29,30,"pl-s"],[61,62,"pl-s"],[66,67,"pl-s"]],[],[[0,1,"pl-v"],[62,63,"pl-s"],[68,69,"pl-s"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[43,44,"pl-s"],[61,62,"pl-s"],[62,63,"pl-s"],[63,106,"pl-corl"],[106,107,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[27,28,"pl-s"],[40,41,"pl-s"]],[],[[0,11,"pl-mh"],[4,11,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[],[],[[0,14,"pl-mh"],[4,14,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"],[10,11,"pl-s"],[44,45,"pl-s"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[0,12,"pl-mh"],[4,12,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[33,34,"pl-s"],[40,41,"pl-s"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,23,"pl-mh"],[4,23,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[15,16,"pl-s"],[22,23,"pl-s"]],[[4,5,"pl-v"],[10,62,"pl-corl"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,16,"pl-mh"],[4,16,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[8,9,"pl-s"],[21,22,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,20,"pl-mh"],[4,20,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[0,22,"pl-mh"],[4,22,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[21,22,"pl-s"]],[[0,1,"pl-v"],[56,57,"pl-s"],[63,64,"pl-s"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,14,"pl-mh"],[4,14,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[],[[0,16,"pl-mh"],[4,16,"pl-en"]],[],[[0,1,"pl-v"],[61,62,"pl-s"],[68,69,"pl-s"]],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-s"],[3,4,"pl-v"]],[[2,3,"pl-s"],[3,4,"pl-v"]],[[2,3,"pl-s"],[3,4,"pl-v"]],[[2,3,"pl-s"],[3,4,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[62,63,"pl-s"],[76,77,"pl-s"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[45,46,"pl-s"],[52,53,"pl-s"]],[[4,5,"pl-v"]],[[0,1,"pl-v"],[17,18,"pl-s"],[36,37,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[0,30,"pl-mh"],[4,30,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[44,45,"pl-s"],[56,57,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"],[22,23,"pl-s"],[27,28,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[30,31,"pl-s"],[35,36,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[50,51,"pl-s"],[70,71,"pl-s"]],[[4,5,"pl-v"],[50,51,"pl-s"],[76,77,"pl-s"]],[[4,5,"pl-v"],[50,51,"pl-s"],[53,54,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[0,35,"pl-mh"],[4,35,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,50,"pl-corl"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[19,20,"pl-s"],[34,35,"pl-s"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,70,"pl-corl"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"],[15,16,"pl-s"],[19,20,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"],[13,14,"pl-s"],[19,20,"pl-s"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[59,60,"pl-s"],[66,67,"pl-s"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[0,30,"pl-mh"],[4,30,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"],[4,5,"pl-s"],[20,21,"pl-s"],[21,22,"pl-s"],[22,85,"pl-corl"],[85,86,"pl-s"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"],[31,32,"pl-s"],[38,39,"pl-s"],[44,45,"pl-s"],[54,55,"pl-s"]],[],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[8,9,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[10,11,"pl-v"]],[[10,11,"pl-v"]],[[10,11,"pl-v"]],[[8,9,"pl-v"]],[[10,11,"pl-v"]],[[10,11,"pl-v"]],[[8,9,"pl-v"]],[[10,11,"pl-v"]],[[10,11,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,7,"pl-s"],[13,14,"pl-s"],[14,15,"pl-s"],[15,80,"pl-corl"],[80,81,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"],[6,7,"pl-s"],[18,19,"pl-s"],[19,20,"pl-s"],[20,124,"pl-corl"],[124,125,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[0,14,"pl-mh"],[4,14,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[20,21,"pl-s"],[27,28,"pl-s"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[8,9,"pl-v"]],[[6,7,"pl-v"]],[],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[],[],[],[],[],[],[],[],[],[],[],[],[[0,22,"pl-mh"],[3,22,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[],[[0,53,"pl-mh"],[4,53,"pl-en"]],[],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,19,"pl-mh"],[4,19,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"],[20,21,"pl-s"],[24,25,"pl-s"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[24,25,"pl-s"],[31,32,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[],[[0,15,"pl-mh"],[4,15,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,7,"pl-s"],[41,42,"pl-s"],[42,43,"pl-s"],[43,89,"pl-corl"],[89,90,"pl-s"]],[[4,5,"pl-v"]],[],[[0,25,"pl-mh"],[4,25,"pl-en"]],[],[[0,1,"pl-v"],[68,69,"pl-s"],[76,77,"pl-s"]],[],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[3,4,"pl-v"]],[[3,4,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"],[8,9,"pl-s"],[24,25,"pl-s"],[32,33,"pl-s"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[8,9,"pl-s"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[8,9,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[8,9,"pl-v"]],[[8,9,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"],[6,7,"pl-s"],[22,23,"pl-s"],[23,24,"pl-s"],[24,99,"pl-corl"],[99,100,"pl-s"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[],[[0,11,"pl-mh"],[4,11,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[0,18,"pl-mh"],[4,18,"pl-en"]],[],[[0,1,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[0,14,"pl-mh"],[4,14,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[0,12,"pl-mh"],[4,12,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,84,"pl-corl"]],[[2,3,"pl-v"]],[],[],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[9,10,"pl-s"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"],[19,20,"pl-s"],[32,33,"pl-s"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[],[[2,3,"pl-v"]],[[7,8,"pl-s"],[15,16,"pl-s"]],[[4,5,"pl-v"]],[],[[0,17,"pl-mh"],[4,17,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[0,1,"pl-v"],[32,33,"pl-s"],[37,38,"pl-s"]],[[2,3,"pl-v"],[44,45,"pl-s"],[49,50,"pl-s"]],[],[[0,21,"pl-mh"],[4,21,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[53,54,"pl-s"],[57,58,"pl-s"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-s"],[11,12,"pl-s"]],[[2,3,"pl-v"]],[],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"],[12,13,"pl-s"],[15,16,"pl-s"]],[[4,5,"pl-v"],[18,19,"pl-s"],[22,23,"pl-s"]],[[4,5,"pl-v"]],[],[[0,17,"pl-mh"],[4,17,"pl-en"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"],[46,47,"pl-s"],[51,52,"pl-s"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"],[6,52,"pl-corl"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[6,7,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[6,7,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[4,5,"pl-v"]],[],[],[[0,9,"pl-mh"],[3,9,"pl-en"]],[],[],[],[],[],[],[],[],[],[[0,18,"pl-mh"],[3,18,"pl-en"]],[],[[0,10,"pl-mh"],[4,10,"pl-en"]],[],[[0,1,"pl-v"],[76,105,"pl-corl"]],[[0,1,"pl-v"],[61,151,"pl-corl"],[152,191,"pl-corl"]],[[0,1,"pl-v"],[122,170,"pl-corl"]],[[0,1,"pl-v"],[74,174,"pl-corl"]],[[0,1,"pl-v"],[42,74,"pl-corl"]],[],[[0,9,"pl-mh"],[4,9,"pl-en"]],[],[[0,1,"pl-v"],[21,22,"pl-s"]],[[14,15,"pl-s"],[15,16,"pl-s"],[16,42,"pl-corl"],[42,43,"pl-s"]],[],[[0,1,"pl-v"]],[],[[0,1,"pl-v"]],[[2,3,"pl-v"]],[[2,3,"pl-v"]],[[4,5,"pl-v"]],[[2,3,"pl-v"]]],"colorizedLines":null}},"title":"distsys-class/README.markdown at master · aphyr/distsys-class","appPayload":{},"meta":{"title":"distsys-class/README.markdown at master · aphyr/distsys-class"}}</script>
<div data-target="react-app.reactRoot"><meta name="github-code-view-meta-stats" id="github-code-view-meta-stats" data-hydrostats="publish"/> <!-- --> <a hidden="" id="code-view-repo-link" href="/aphyr/distsys-class" data-discover="true"></a> <div class="d-none"></div><div><div style="--spacing:var(--spacing-none)" class="prc-PageLayout-PageLayoutRoot--KH-d" data-component="SplitPageLayout" data-has-sidebar="true"><div class="prc-PageLayout-SidebarWrapper-kLG4B CopilotSidePanelSidebar-module__SidePanel__L3O0C CopilotSidePanelSidebar-module__HiddenSidePanel__TBRGn" style="--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="end" data-sticky="true" data-responsive-variant="fullscreen"><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-SidebarVerticalDivider-0Rl0V" data-component="PageLayout.VerticalDivider" data-variant="line" data-position="end" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="450" aria-valuemax="768" aria-valuenow="544" aria-valuetext="Pane width 544 pixels" tabindex="0"></div></div><div class="prc-PageLayout-Sidebar-iciWg" data-component="SplitPageLayout.Sidebar" data-resizable="true" style="--spacing:var(--spacing-normal);--pane-min-width:450px;--pane-max-width:768px;--pane-width-custom:544px;--pane-width-size:var(--pane-width-custom);--pane-width:544px"><div class="height-full" data-testid="copilot-code-view-side-panel"><div id="copilot-side-panel-content" class="height-full"></div></div></div></div><div class="prc-PageLayout-PageLayoutWrapper-2BhU2" data-width="full"><div class="prc-PageLayout-PageLayoutContent-BneH9"><div id="repos-file-tree-sidebar" class="CodeViewFileTreeLayout-module__sidebar__n_Aau" tabindex="0"><div class="prc-PageLayout-PaneWrapper-pHPop ReposFileTreePane-module__Pane__rBZpI ReposFileTreePane-module__HideTree__AYZnm ReposFileTreePane-module__HidePane__VHAVt" style="--offset-header:0px;--spacing-row:var(--spacing-none);--spacing-column:var(--spacing-none)" data-is-hidden="false" data-position="start" data-sticky="true"><div class="prc-PageLayout-HorizontalDivider-JLVqp prc-PageLayout-PaneHorizontalDivider-9tbnE" data-component="PageLayout.HorizontalDivider" data-variant-regular="none" data-variant-narrow="none" data-position="start" style="--spacing-divider:var(--spacing-none);--spacing:var(--spacing-none)"></div><div class="prc-PageLayout-Pane-AyzHK" data-component="SplitPageLayout.Pane" data-resizable="true" style="--spacing:var(--spacing-none);--pane-min-width:256px;--pane-max-width:calc(100vw - var(--pane-max-width-diff));--pane-width-size:var(--pane-width-large);--pane-width:320px"></div><div class="prc-PageLayout-VerticalDivider-9QRmK prc-PageLayout-PaneVerticalDivider-le57g" data-component="PageLayout.VerticalDivider" data-variant-narrow="none" data-variant-regular="line" data-variant-wide="line" data-position="start" style="--spacing:var(--spacing-none)"><div class="prc-PageLayout-DraggableHandle-9s6B4" data-component="PageLayout.DragHandle" role="slider" aria-label="Draggable pane splitter" aria-valuemin="256" aria-valuemax="600" aria-valuenow="320" aria-valuetext="Pane width 320 pixels" tabindex="0"></div></div></div></div><div data-component="SplitPageLayout.Content" class="prc-PageLayout-ContentWrapper-gR9eG"><div class="prc-PageLayout-Content-xWL-A" data-width="full" style="--spacing:var(--spacing-none)"><div class="SharedPageLayout-module__content__IwGAp" data-selector="repos-split-pane-content" id="repos-split-pane-content" tabindex="0"> <!-- --> <div class="container CodeViewHeader-module__Box__JkPOb"><div class="CodeViewHeader-module__StickyHeader__Qn7UN" id="StickyHeader"><div class="CodeViewHeader-module__Box_1__SbNDV"><div class="CodeViewHeader-module__Box_2__TB46f"><div class="react-code-view-header-wrap--narrow CodeViewHeader-module__Box_3__q1zUL"><div class="CodeViewHeader-module__treeToggleWrapper__RQ__9"><h2 class="use-tree-pane-module__Heading__s4QbZ prc-Heading-Heading-MtWFE" data-component="Heading"><button data-component="Button" type="button" aria-label="Expand file tree" data-testid="expand-file-tree-button-mobile" class="prc-Button-ButtonBase-9n-Xk ExpandFileTreeButton-module__Button_1__Svs95" data-loading="false" data-size="medium" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-left" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M7.78 12.53a.75.75 0 0 1-1.06 0L2.47 8.28a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L4.81 7h7.44a.75.75 0 0 1 0 1.5H4.81l2.97 2.97a.75.75 0 0 1 0 1.06Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Files</span></span></button><button data-component="IconButton" type="button" data-testid="expand-file-tree-button" aria-controls="repos-file-tree" class="prc-Button-ButtonBase-9n-Xk position-relative ExpandFileTreeButton-module__expandButton__hDOcv ExpandFileTreeButton-module__filesButtonBreakpoint__zEvz3 fgColor-muted prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-labelledby="_R_4lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-sidebar-collapse" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M6.823 7.823a.25.25 0 0 1 0 .354l-2.396 2.396A.25.25 0 0 1 4 10.396V5.604a.25.25 0 0 1 .427-.177Z"></path><path d="M1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0ZM1.5 1.75v12.5c0 .138.112.25.25.25H9.5v-13H1.75a.25.25 0 0 0-.25.25ZM11 14.5h3.25a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25H11Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="se" data-component="Tooltip" aria-hidden="true" id="_R_4lla9lik5_">Expand file tree</span><div class="d-none"></div></h2></div><div class="react-code-view-header-mb--narrow mr-2"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="master branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector-wide"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">master</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="react-code-view-header-mb--narrow CodeViewHeader-module__Box_5__MQ0hL"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__lg__Rjz0A"><nav data-testid="breadcrumbs" aria-labelledby="repos-header-breadcrumb-heading" id="repos-header-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="repos-header-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/aphyr/distsys-class/tree/master" data-discover="true">distsys-class</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__lg__Rjz0A" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__lg__Rjz0A prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="file-name-id">README.markdown</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lla9lik5_">Copy path</span></div></div></div><div class="react-code-view-header-element--wide"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><div><div class="CodeViewHeader-module__FileResultsList__JDzUy"><span class="d-flex FileResultsList-module__FilesSearchBox__ivVkc TextInput-wrapper prc-components-TextInputWrapper-Hpdqi prc-components-TextInputBaseWrapper-wY-n0" data-no-trailing-action="true" data-component="TextInput" data-leading-visual="true" data-trailing-visual="true" aria-busy="false"><span class="TextInput-icon" id="_R_5ipla9lik5_" aria-hidden="true" data-component="TextInput.LeadingVisual"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-search" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg></span><input type="text" aria-label="Go to file" role="combobox" aria-controls="file-results-list" aria-expanded="false" aria-haspopup="dialog" autoCorrect="off" spellCheck="false" placeholder="Go to file" aria-describedby="_R_5ipla9lik5_ _R_5ipla9lik5H1_" data-component="input" class="prc-components-Input-IwWrt" value=""/><span class="TextInput-icon" id="_R_5ipla9lik5H1_" aria-hidden="true" data-component="TextInput.TrailingVisual"></span></span></div><div class="d-none"></div></div><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-wide" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4pla9lik5_" id="_R_14pla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4pla9lik5_">More file actions</span></div></div></div><div class="react-code-view-header-element--narrow"><div class="CodeViewHeader-module__Box_7___0R6c"><div class="d-flex gap-2"><button data-component="Button" type="button" style="display:none" class="prc-Button-ButtonBase-9n-Xk NavigationMenu-module__Button__LpKgm" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Blame</span></span></button><div class="d-none"></div><button data-component="IconButton" type="button" data-testid="more-file-actions-button-nav-menu-narrow" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click NavigationMenu-module__IconButton__HpX3G prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-labelledby="_R_v4tla9lik5_" id="_R_14tla9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_v4tla9lik5_">More file actions</span></div></div></div></div></div></div></div><div class="CodeView-module__contentWrapper__cG2JH"><div class="react-code-view-bottom-padding"><div class="BlobTopBanners-module__Box__v_nvx"></div></div> <div class="d-none"></div><div class="d-flex flex-column border rounded-2 tmp-mb-3 pl-1"><div class="LatestCommit-module__Box__B25ZT"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">Latest commit</h2><div style="width:120px" class="Skeleton Skeleton--text" data-testid="loading"> </div><div class="d-flex flex-shrink-0 gap-2"><div data-testid="latest-commit-details" class="d-none d-sm-flex flex-items-center"></div><div class="d-flex gap-2"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">History</h2><a data-component="LinkButton" href="/aphyr/distsys-class/commits/master/README.markdown" class="prc-Button-ButtonBase-9n-Xk d-none d-lg-flex LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><span class="fgColor-default">History</span></span></span></a><div class="d-sm-none"></div><div class="d-flex d-lg-none"><a data-component="LinkButton" aria-label="View commit history for this file." href="/aphyr/distsys-class/commits/master/README.markdown" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby="_R_4mlala9lik5_"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span></span></a><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" role="tooltip" aria-hidden="true" id="_R_4mlala9lik5_">History</span></div></div></div></div></div><div class="d-flex flex-row"><div class="container BlobViewContent-module__blobContainer__DtH2d"><div class="react-code-size-details-banner BlobViewContent-module__codeSizeDetails__e5sUw"><div class="react-code-size-details-banner CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="78.5 KB" style="--truncate-max-width:100%"><span>1894 lines (1661 loc) · 78.5 KB</span></div></div></div></div><div class="react-blob-view-header-sticky BlobViewContent-module__stickyHeader__VwxB5" id="repos-sticky-header"><div class="BlobViewHeader-module__Box__yhm9u"><div class="react-blob-sticky-header"><div class="FileNameStickyHeader-module__outerWrapper__ZL4Xc FileNameStickyHeader-module__outerWrapperHidden__Zpynk"><div class="FileNameStickyHeader-module__Box_1__Hazu5"><div class="FileNameStickyHeader-module__Box_2__hoolP"><div class="FileNameStickyHeader-module__Box_3__MVKsk"><button data-component="Button" type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="master branch" data-testid="anchor-button" data-icv-name="Switch branches/tags" class="prc-Button-ButtonBase-9n-Xk ref-selector-class RefSelectorAnchoredOverlay-module__RefSelectorOverlayBtn__a3WK3" data-loading="false" data-size="medium" data-variant="default" id="ref-picker-repos-header-ref-selector"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3"><div class="RefSelectorAnchoredOverlay-module__RefSelectorOverlayContainer__yaf4p"><div style="max-width:125px" class="ref-selector-button-text-container RefSelectorAnchoredOverlay-module__RefSelectorBtnTextContainer__Di3rk"><span class="RefSelectorAnchoredOverlay-module__RefSelectorText__w_fmP">master</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-YNt2F prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="d-none"></div></div><div class="FileNameStickyHeader-module__Box_4__FLhtt"><div class="Breadcrumb-module__container__Vxvev Breadcrumb-module__md__Wb1Gs"><nav data-testid="breadcrumbs" aria-labelledby="sticky-breadcrumb-heading" id="sticky-breadcrumb" class="Breadcrumb-module__nav__rQFDj"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading" id="sticky-breadcrumb-heading">Breadcrumbs</h2><ol class="Breadcrumb-module__list__ZH6zr"><li class="Breadcrumb-module__listItem__Ib0x_"><a class="Breadcrumb-module__repoLink__O2Nbs prc-Link-Link-9ZwDx" data-component="Link" data-testid="breadcrumbs-repo-link" href="/aphyr/distsys-class/tree/master" data-discover="true">distsys-class</a></li></ol></nav><div data-testid="breadcrumbs-filename" class="Breadcrumb-module__filename__equZR"><span class="Breadcrumb-module__separator__eNwsI Breadcrumb-module__md__Wb1Gs" aria-hidden="true">/</span><h1 class="Breadcrumb-module__filenameHeading__MNMtw Breadcrumb-module__md__Wb1Gs prc-Heading-Heading-MtWFE" data-component="Heading" tabindex="-1" id="sticky-file-name-id">README.markdown</h1></div><button data-component="IconButton" type="button" class="prc-Button-ButtonBase-9n-Xk ml-2 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_7lcpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="CopyToClipboardIconButton-module__tooltip__WyiwL prc-TooltipV2-Tooltip-tLeuB" data-direction="s" data-component="Tooltip" aria-label="Copy path" aria-hidden="true" id="_R_7lcpala9lik5_">Copy path</span></div></div></div><button data-component="Button" type="button" class="prc-Button-ButtonBase-9n-Xk FileNameStickyHeader-module__Button__LSEU_ FileNameStickyHeader-module__GoToTopButton__nxAFn" data-loading="false" data-size="small" data-variant="invisible"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="leadingVisual" class="prc-Button-Visual-YNt2F prc-Button-LeadingVisual-UySKu prc-Button-VisualWrap-E4cnq"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-arrow-up" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M3.47 7.78a.75.75 0 0 1 0-1.06l4.25-4.25a.75.75 0 0 1 1.06 0l4.25 4.25a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018L9 4.81v7.44a.75.75 0 0 1-1.5 0V4.81L4.53 7.78a.75.75 0 0 1-1.06 0Z"></path></svg></span><span data-component="text" class="prc-Button-Label-FWkx3">Top</span></span></button></div></div></div><div class="BlobViewHeader-module__Box_1__VEmuQ"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone__rwWIk prc-Heading-Heading-MtWFE" data-component="Heading" data-testid="screen-reader-heading">File metadata and controls</h2><div class="BlobViewHeader-module__Box_2__icUs2"><ul aria-label="File view" class="prc-SegmentedControl-SegmentedControl-lqIXp BlobTabButtons-module__SegmentedControl__jen2u" data-variant="default" data-size="small" data-component="SegmentedControl"><li class="prc-SegmentedControl-Item-tSCQh" data-selected="" data-component="SegmentedControl.Button"><button aria-pressed="true" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:transparent"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Preview">Preview</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Code">Code</div></span></button></li><li class="prc-SegmentedControl-Item-tSCQh" data-component="SegmentedControl.Button"><button aria-pressed="false" class="prc-SegmentedControl-Button-E48xz" type="button" style="--separator-color:var(--borderColor-default)"><span class="prc-SegmentedControl-Content-1COlk segmentedControl-content"><div class="prc-SegmentedControl-Text-7S2y2 segmentedControl-text" data-text="Blame">Blame</div></span></button></li></ul><div class="d-none"></div><div class="react-code-size-details-in-header CodeSizeDetails-module__Box__VcD6l"><div class="text-mono CodeSizeDetails-module__Box_1__GVxQL"><div data-testid="blob-size" class="CodeSizeDetails-module__Truncate_1__lE93V prc-Truncate-Truncate-2G1eo" data-inline="true" title="78.5 KB" style="--truncate-max-width:100%"><span>1894 lines (1661 loc) · 78.5 KB</span></div></div></div></div><div class="BlobViewHeader-module__Box_3__ng6v2"><div class="d-none"></div><div class="react-blob-header-edit-and-raw-actions BlobViewHeader-module__Box_4__J4Y4W"><div class="d-none"></div><div class="prc-ButtonGroup-ButtonGroup-vFUrY" data-component="ButtonGroup"><div class="prc-ButtonGroup-Item-PqvDl"><a data-component="LinkButton" href="https://github.com/aphyr/distsys-class/raw/refs/heads/master/README.markdown" data-testid="raw-button" class="prc-Button-ButtonBase-9n-Xk LinkButton-module__linkButton__nFnov BlobViewHeader-module__LinkButton__X9kx2" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-Iohp5"><span data-component="text" class="prc-Button-Label-FWkx3">Raw</span></span></a></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="copy-raw-button" class="prc-Button-ButtonBase-9n-Xk prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_qaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-copy" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_qaucpala9lik5_">Copy raw file</span></div><div class="prc-ButtonGroup-Item-PqvDl"><button data-component="IconButton" type="button" data-testid="download-raw-button" class="prc-Button-ButtonBase-9n-Xk BlobViewHeader-module__downloadButton__ef459 prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="default" aria-labelledby="_R_eaucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-download" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2.75 14A1.75 1.75 0 0 1 1 12.25v-2.5a.75.75 0 0 1 1.5 0v2.5c0 .138.112.25.25.25h10.5a.25.25 0 0 0 .25-.25v-2.5a.75.75 0 0 1 1.5 0v2.5A1.75 1.75 0 0 1 13.25 14Z"></path><path d="M7.25 7.689V2a.75.75 0 0 1 1.5 0v5.689l1.97-1.969a.749.749 0 1 1 1.06 1.06l-3.25 3.25a.749.749 0 0 1-1.06 0L4.22 6.78a.749.749 0 1 1 1.06-1.06l1.97 1.969Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_eaucpala9lik5_">Download raw file</span></div></div></div><button data-component="IconButton" type="button" aria-pressed="false" class="prc-Button-ButtonBase-9n-Xk tmp-mr-2 TableOfContents-module__IconButton__jrlNM prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_3ucpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-list-unordered" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.75 2.5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2 14a1 1 0 1 1 0-2 1 1 0 0 1 0 2Zm1-6a1 1 0 1 1-2 0 1 1 0 0 1 2 0ZM2 4a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="n" data-component="Tooltip" aria-hidden="true" id="_R_3ucpala9lik5_">Outline</span><div class="react-blob-header-edit-and-raw-actions-combined"><button data-component="IconButton" type="button" title="More file actions" data-testid="more-file-actions-button" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-9n-Xk js-blob-dropdown-click BlobViewHeader-module__IconButton__XrMQY prc-Button-IconButton-fyge7" data-loading="false" data-no-visuals="true" data-size="small" data-variant="invisible" aria-labelledby="_R_fkecpala9lik5_" id="_R_kecpala9lik5_"><svg data-component="Octicon" aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button><span class="prc-TooltipV2-Tooltip-tLeuB" data-direction="nw" data-component="Tooltip" aria-hidden="true" id="_R_fkecpala9lik5_">Edit and raw actions</span></div></div></div></div><div></div></div><div class="BlobViewContent-module__blobContentWrapper__JS0W6"><section aria-labelledby="file-name-id-wide file-name-id-mobile" class="BlobContent-module__blobContentSection__VOgZq BlobContent-module__blobContentSectionMarkdown__mPLOK" style="margin-top:46px"><div class="js-snippet-clipboard-copy-unpositioned BlobContent-module__markdownBlob__T8jpG" data-hpc="true" containertiming="hpc"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">An Introduction to Distributed Systems</h1><a id="user-content-an-introduction-to-distributed-systems" class="anchor" aria-label="Permalink: An Introduction to Distributed Systems" href="#an-introduction-to-distributed-systems"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Copyright Kyle Kingsbury &amp; Jepsen, LLC.</p>
<p dir="auto">This outline accompanies a 16 to 32 hour <a href="https://jepsen.io/training.html" rel="nofollow">overview class on distributed systems
fundamentals</a>. The course aims to introduce
software engineers to the practical basics of distributed systems, through
lecture and discussion, and optional
<a href="https://github.com/jepsen-io/maelstrom">labwork</a>. Participants will gain an
intuitive understanding of key distributed systems terms, an overview of the
algorithmic landscape, and explore production concerns.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Intro</h2><a id="user-content-intro" class="anchor" aria-label="Permalink: Intro" href="#intro"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>My name is Kyle Kingsbury
<ul dir="auto">
<li>Pronouns: he/him</li>
<li>Email: <a href="mailto:aphyr@jepsen.io">aphyr@jepsen.io</a></li>
<li>This outline: <a href="https://github.com/aphyr/distsys-class">https://github.com/aphyr/distsys-class</a></li>
</ul>
</li>
<li>I've worked on distributed systems from several perspectives
<ul dir="auto">
<li>2003--2009: sysadmin, network operations, web developer</li>
<li>2009--2014: backend engineer, database wrangler at various web startups</li>
<li>2014--now: distributed systems safety testing (Jepsen)</li>
</ul>
</li>
<li>This class is aimed at practitioners
<ul dir="auto">
<li>"Things I Wish I Would Have Known"</li>
<li>Backend engineers</li>
<li>Frontend engineers</li>
<li>Ops folks</li>
<li>Product managers</li>
</ul>
</li>
<li>This class aims to prepare you to write, operate, and use distributed systems
<ul dir="auto">
<li>First half: theoretical framework
<ul dir="auto">
<li>A map of how concepts fit together
<ul dir="auto">
<li>Not about memorizing everything; it's about knowing where to look</li>
</ul>
</li>
<li>Establish a shared language
<ul dir="auto">
<li>To talk with your peers</li>
<li>To read a paper</li>
<li>To evaluate a system's claims</li>
</ul>
</li>
<li>Understand fundamental principles
<ul dir="auto">
<li>Time, ordering, nodes, networks, faults, liveness, safety</li>
</ul>
</li>
<li>Classes of algorithms
<ul dir="auto">
<li>What they can and can't do, when to apply each</li>
</ul>
</li>
</ul>
</li>
<li>Second half: practical concerns
<ul dir="auto">
<li>A grab bag of design patterns
<ul dir="auto">
<li>Illustrative anecdotes</li>
<li>Computers were, in fact, a mistake</li>
<li>What <em>isn't</em> in the papers?</li>
</ul>
</li>
<li>Finally, production concerns
<ul dir="auto">
<li>Deployment, debugging, monitoring</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li>Class logistics
<ul dir="auto">
<li>This class is what you make of it
<ul dir="auto">
<li>We can go as deep as you want, or skip over familiar ground</li>
<li>Jump in with questions, challenges, ideas at any time</li>
</ul>
</li>
<li>We'll take ~10-minute breaks every hour or so, plus lunch</li>
<li>If you need to duck out for whatever reason, that's fine!
<ul dir="auto">
<li>Outages, kids, pets, whatever</li>
<li>The outline on GitHub can help fill in the gaps</li>
<li>You can ask me during class or a break about something you missed!</li>
</ul>
</li>
<li>I'll ask for your thoughts often, but you can also jump in whenever
<ul dir="auto">
<li>If you <em>don't</em> want to be called on, that's cool--please let me know</li>
</ul>
</li>
<li>If you think of something after class, you can email me!
<ul dir="auto">
<li>Or write it down for the next day's discussion</li>
</ul>
</li>
<li>Recording is prohibited
<ul dir="auto">
<li>Teaching is my livelihood</li>
<li>And this outline is free!</li>
</ul>
</li>
</ul>
</li>
<li>If we're doing labs:
<ul dir="auto">
<li>There are labs for this class!
<ul dir="auto">
<li>Mostly in the middle</li>
</ul>
</li>
<li>Remind folks of the prereq: <a href="https://github.com/jepsen-io/maelstrom/blob/main/doc/01-getting-ready/index.md">Getting Ready</a></li>
</ul>
</li>
<li>If we're remote:
<ul dir="auto">
<li>You can turn off video whenever you like
<ul dir="auto">
<li>But please, if <em>some</em> people can be on video, that's really helpful</li>
<li>Seeing faces helps me know whether the lecture is working for you!</li>
</ul>
</li>
<li>For the labs, we'll be doing a shared tmux session</li>
</ul>
</li>
<li>OK, let's get going!</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">What makes a thing distributed?</h2><a id="user-content-what-makes-a-thing-distributed" class="anchor" aria-label="Permalink: What makes a thing distributed?" href="#what-makes-a-thing-distributed"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Lamport, 1987:</p>
<blockquote>
<p dir="auto">A distributed system is one in which the failure of a computer
you didn't even know existed can render your own computer
unusable.</p>
</blockquote>
<ul dir="auto">
<li>First glance: *nix boxen in our colo, running processes communicating via
TCP or UDP.
<ul dir="auto">
<li>Or boxes in EC2, Rackspace, etc</li>
<li>Maybe communicating over InfiniBand</li>
<li>Separated by inches and a LAN</li>
<li>Or by kilometers and the internet</li>
</ul>
</li>
<li>Most mobile apps are also taking part in a distributed system
<ul dir="auto">
<li>Communicating over a truly awful network</li>
<li>Same goes for desktop web browsers</li>
<li>It's not just servers--it's clients too!</li>
</ul>
</li>
<li>More generally: distributed systems are
<ul dir="auto">
<li>Made up of parts which interact</li>
<li>Slowly</li>
<li>And often unreliably</li>
<li>Whatever those mean for you</li>
</ul>
</li>
<li>So also:
<ul dir="auto">
<li>Redundant CPUs in an airplane</li>
<li>ATMs and Point-of-Sale terminals</li>
<li>Space probes</li>
<li>Paying bills</li>
<li>Doctors making referrals</li>
<li>Trying to make plans via text message</li>
<li>Every business meeting ever</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Nodes and networks</h2><a id="user-content-nodes-and-networks" class="anchor" aria-label="Permalink: Nodes and networks" href="#nodes-and-networks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>We call each part of a distributed system a <em>node</em>
<ul dir="auto">
<li>Also known as <em>processes</em>, <em>agents</em>, or <em>actors</em></li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Nodes</h3><a id="user-content-nodes" class="anchor" aria-label="Permalink: Nodes" href="#nodes"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Characteristic latency
<ul dir="auto">
<li>Operations inside a node are "fast"</li>
<li>Operations between nodes are "slow"</li>
<li>What's fast or slow depends on what the system does</li>
</ul>
</li>
<li>Nodes are reliable
<ul dir="auto">
<li>Fail as a unit</li>
<li>You know when problems occur</li>
<li>State is coherent</li>
<li>State transitions occur in a nice, orderly fashion</li>
<li>Typically modeled as some kind of single-threaded state machine</li>
</ul>
</li>
<li>A node could <em>itself</em> be a distributed system
<ul dir="auto">
<li>But so long as that system as a whole provides "fast, coherent"
operations, we can treat it as a single node.</li>
</ul>
</li>
<li>Formal models for processes
<ul dir="auto">
<li><a href="https://dspace.mit.edu/handle/1721.1/6952" rel="nofollow">Actor model</a></li>
<li><a href="https://www.cs.cmu.edu/~crary/819-f09/Hoare78.pdf" rel="nofollow">Communicating Sequential Processes</a></li>
<li><a href="https://golem.ph.utexas.edu/category/2009/08/the_pi_calculus.html" rel="nofollow">Pi-calculus</a></li>
<li><a href="http://lucacardelli.name/Talks/1998-03-30%20Mobile%20Ambients%20%28ETAPS%29.pdf" rel="nofollow">Ambient calculus</a></li>
</ul>
</li>
<li>Formal models for node failure
<ul dir="auto">
<li>Crash-stop</li>
<li>Crash-recover</li>
<li>Crash-amnesia</li>
<li>Byzantine</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Networks as message flows</h3><a id="user-content-networks-as-message-flows" class="anchor" aria-label="Permalink: Networks as message flows" href="#networks-as-message-flows"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Nodes interact via a <em>network</em>
<ul dir="auto">
<li>Humans interact via spoken words</li>
<li>Particles interact via fields</li>
<li>Computers interact via IP, or UDP, or SCTP, or ...</li>
</ul>
</li>
<li>We model those interactions as discrete <em>messages</em> sent between nodes</li>
<li>Messages take <em>time</em> to propagate
<ul dir="auto">
<li>This is the "slow" part of the distributed system</li>
<li>We call this "latency"</li>
</ul>
</li>
<li>Messages can often be lost
<ul dir="auto">
<li>This is another "unreliable" part of the distributed system</li>
</ul>
</li>
<li>Network is rarely homogenous
<ul dir="auto">
<li>Some links slower/smaller/more-likely-to-fail than others</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Causality diagrams</h3><a id="user-content-causality-diagrams" class="anchor" aria-label="Permalink: Causality diagrams" href="#causality-diagrams"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>We can represent the interaction of nodes and the network as a diagram
<ul dir="auto">
<li>Time flows left-to-right, or top-to-bottom</li>
<li>Nodes are lines in the direction of time (because they stay in place)</li>
<li>Messages as slanted paths <em>connecting</em> nodes</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Synchronous networks</h3><a id="user-content-synchronous-networks" class="anchor" aria-label="Permalink: Synchronous networks" href="#synchronous-networks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Nodes execute in lockstep: time between node steps is always 1.</li>
<li>Message delay is bounded</li>
<li>Effectively a perfect global clock</li>
<li>Easy to prove stuff about
<ul dir="auto">
<li>You probably don't have one</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Semi-synchronous networks</h3><a id="user-content-semi-synchronous-networks" class="anchor" aria-label="Permalink: Semi-synchronous networks" href="#semi-synchronous-networks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Like synchronous, but the clock is only approximate, e.g. in [c, 1]</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Asynchronous networks</h3><a id="user-content-asynchronous-networks" class="anchor" aria-label="Permalink: Asynchronous networks" href="#asynchronous-networks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Execute independently, whenever: step time is anywhere in [0, 1]</li>
<li>Unbounded message delays</li>
<li>No global clocks</li>
<li>Weaker than semi- or synchronous networks
<ul dir="auto">
<li>Implies certain algorithms can't be as efficient</li>
<li>Implies certain algorithms are <em>impossible</em></li>
<li>See e.g. Attiya &amp; Mavronicolas, "Efficiency of Semi-Synchronous vs
Asynchronous Networks"</li>
</ul>
</li>
<li>IP networks are definitely asynchronous
<ul dir="auto">
<li>But <em>in practice</em> the really pathological stuff doesn't happen</li>
<li>Most networks recover in seconds to weeks, not "never"
<ul dir="auto">
<li>Conversely, human timescales are on the orders of seconds to weeks</li>
<li>So we can't pretend the problems don't exist</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">When networks go wrong</h2><a id="user-content-when-networks-go-wrong" class="anchor" aria-label="Permalink: When networks go wrong" href="#when-networks-go-wrong"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Asynchronous networks are allowed to
<ul dir="auto">
<li>Duplicate</li>
<li>Delay</li>
<li>Drop</li>
<li>Reorder</li>
</ul>
</li>
<li>Drops and delays are indistinguishable</li>
<li>Byzantine networks are allowed to mess with messages <em>arbitrarily</em>
<ul dir="auto">
<li>Including rewriting their content</li>
<li>They mostly don't happen in real networks
<ul dir="auto">
<li>Mostly</li>
<li><a href="https://www.pagerduty.com/blog/the-discovery-of-apache-zookeepers-poison-packet/" rel="nofollow">https://www.pagerduty.com/blog/the-discovery-of-apache-zookeepers-poison-packet/</a></li>
<li><a href="https://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19" rel="nofollow">https://tech.vijayp.ca/linux-kernel-bug-delivers-corrupt-tcp-ip-data-to-mesos-kubernetes-docker-containers-4986f88f7a19</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Low level protocols</h2><a id="user-content-low-level-protocols" class="anchor" aria-label="Permalink: Low level protocols" href="#low-level-protocols"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">TCP</h3><a id="user-content-tcp" class="anchor" aria-label="Permalink: TCP" href="#tcp"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>TCP <em>works</em>. Use it.
<ul dir="auto">
<li>Not perfect; you can go faster</li>
<li>But you'll know when this is the case</li>
</ul>
</li>
<li>In practice, TCP prevents duplicates and reorders in the context of a single
TCP conn
<ul dir="auto">
<li>But you're probably gonna open more than one connection</li>
<li>If for no other reason than TCP conns eventually fail</li>
<li>And when that happens, you'll either a.) have missed messages or b.) retry</li>
<li>You can <em>reconstruct</em> an ordering by encoding your own sequence numbers on
top of TCP</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">UDP</h3><a id="user-content-udp" class="anchor" aria-label="Permalink: UDP" href="#udp"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Same addressing rules as TCP, but no stream invariants</li>
<li>Lots of people want UDP "for speed"
<ul dir="auto">
<li>Don't consider that routers and nodes can and will arbitrarily drop packets</li>
<li>Don't consider that their packets <em>will</em> be duplicated</li>
<li>And reordered</li>
<li>"But at least it's unbiased right?"
<ul dir="auto">
<li>WRONG!</li>
</ul>
</li>
<li>This causes all kinds of havoc in, say, metrics collection</li>
<li>And debugging it is <em>hard</em></li>
<li>TCP gives you flow control and repacks logical messages into packets
<ul dir="auto">
<li>You'll need to re-build flow-control and backpressure</li>
</ul>
</li>
<li>TLS over UDP is a thing, but tough</li>
</ul>
</li>
<li>UDP is really useful where TCP FSM overhead is prohibitive
<ul dir="auto">
<li>Memory pressure</li>
<li>Lots of short-lived conns and socket reuse</li>
</ul>
</li>
<li>Especially useful where best-effort delivery maps well to the system goals
<ul dir="auto">
<li>Voice calls: people will apologize and repeat themselves</li>
<li>Games: stutters and lag, but catch up later</li>
<li>Higher-level protocols impose sanity on underlying chaos</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Clocks</h2><a id="user-content-clocks" class="anchor" aria-label="Permalink: Clocks" href="#clocks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>When a system is split into independent parts, we still want some kind of
<em>order</em> for events</li>
<li>Clocks help us order things: first this, THEN that</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Wall Clocks</h3><a id="user-content-wall-clocks" class="anchor" aria-label="Permalink: Wall Clocks" href="#wall-clocks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>In theory, the operating system clock gives you a partial order on system events
<ul dir="auto">
<li>Caveat: NTP is probably not as good as you think</li>
<li>Caveat: Definitely not well-synced between nodes</li>
<li>Caveat: Hardware can drift
<ul dir="auto">
<li>Multiple reports of supermicro TSCs causing system clock to run 26ms/s fast
<ul dir="auto">
<li><a href="https://groups.google.com/forum/#!search/Supermicro$20SYS-1029UX-LL1-S16$20system$20clock$20running$20too$20fast/mechanical-sympathy/oG9vLZVYjVA/DU-T9QpBAgAJ" rel="nofollow">https://groups.google.com/forum/#!search/Supermicro$20SYS-1029UX-LL1-S16$20system$20clock$20running$20too$20fast/mechanical-sympathy/oG9vLZVYjVA/DU-T9QpBAgAJ</a></li>
</ul>
</li>
</ul>
</li>
<li>Caveat: By <em>centuries</em>
<ul dir="auto">
<li>NTP might not care</li>
<li><a href="http://rachelbythebay.com/w/2017/09/27/2153/" rel="nofollow">http://rachelbythebay.com/w/2017/09/27/2153/</a></li>
</ul>
</li>
<li>Caveat: NTP can still jump the clock backwards (default: delta &gt; 128 ms)
<ul dir="auto">
<li><a href="https://www.eecis.udel.edu/~mills/ntp/html/clock.html" rel="nofollow">https://www.eecis.udel.edu/~mills/ntp/html/clock.html</a></li>
</ul>
</li>
<li>Caveat: POSIX time is not monotonic by <em>definition</em>
<ul dir="auto">
<li>Cloudflare 2017: Leap second at midnight UTC meant time flowed backwards</li>
<li>At the time, Go didn't offer access to CLOCK_MONOTONIC</li>
<li>Computed a negative duration, then fed it to rand.int63n(), which paniced</li>
<li>Caused DNS resolutions to fail: 1% of HTTP requests affected for several hours</li>
<li><a href="https://blog.cloudflare.com/how-and-why-the-leap-second-affected-cloudflare-dns/" rel="nofollow">https://blog.cloudflare.com/how-and-why-the-leap-second-affected-cloudflare-dns/</a></li>
</ul>
</li>
<li>Caveat: The timescales you want to measure may not be attainable</li>
<li>Caveat: Threads can sleep</li>
<li>Caveat: Runtimes can sleep</li>
<li>Caveat: OS's can sleep</li>
<li>Caveat: "Hardware" can sleep</li>
<li>Caveat: The hypervisor can lie to you
<ul dir="auto">
<li>By 16+ seconds in a 15 minute period!?</li>
<li><a href="https://gist.github.com/sandfox/32e749b5eac861c93f1bbeb8782ae8fd">https://gist.github.com/sandfox/32e749b5eac861c93f1bbeb8782ae8fd</a></li>
</ul>
</li>
</ul>
</li>
<li>Just don't.</li>
<li>At least OS monotonic clocks are monotonic, right?
<ul dir="auto">
<li>oh no: <a href="https://github.com/rust-lang/rust/blob/eed12bcd0cb281979c4c9ed956b9e41fda2bfaeb/src/libstd/time.rs#L201-L232">https://github.com/rust-lang/rust/blob/eed12bcd0cb281979c4c9ed956b9e41fda2bfaeb/src/libstd/time.rs#L201-L232</a></li>
<li>Use CLOCK_BOOTTIME, which accounts for VM pauses</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Lamport Clocks</h3><a id="user-content-lamport-clocks" class="anchor" aria-label="Permalink: Lamport Clocks" href="#lamport-clocks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Lamport 1977: "Time, Clocks, and the Ordering of Events in a Distributed System"
<ul dir="auto">
<li>One clock per process</li>
<li>Increments monotonically with each state transition: <code>t' = t + 1</code></li>
<li>Included with every message sent</li>
<li><code>t' = max(t, t_msg + 1)</code></li>
</ul>
</li>
<li>If we have a total ordering of processes, we can impose a total order on
events
<ul dir="auto">
<li>But that order could be pretty unintuitive</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Vector Clocks</h3><a id="user-content-vector-clocks" class="anchor" aria-label="Permalink: Vector Clocks" href="#vector-clocks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Generalizes Lamport clocks to a vector of all process clocks</li>
<li><code>t_i' = max(t_i, t_msg_i)</code></li>
<li>For every operation, increment that process' clock in the vector</li>
<li>Provides a partial causal order
<ul dir="auto">
<li>A &lt; B iff all A_i &lt;= B_i, and at least one A_i &lt; B_i</li>
<li>Specifically, given a pair of events, we can determine causal relationships
<ul dir="auto">
<li>A in causal past of B implies A &lt; B</li>
<li>B in causal past of A implies B &lt; A</li>
<li>Independent otherwise</li>
</ul>
</li>
</ul>
</li>
<li>Pragmatically: the past is shared; the present is independent
<ul dir="auto">
<li>Only "present", independent states need to be preserved</li>
<li>Ancestor states can be discarded</li>
<li>Lets us garbage-collect the past</li>
</ul>
</li>
<li>O(processes) in space
<ul dir="auto">
<li>Requires coordination for GC</li>
<li>Or sacrifice correctness and prune old vclock entries</li>
</ul>
</li>
<li>Variants
<ul dir="auto">
<li>Dotted Version Vectors - for client/server systems, orders <em>more</em> events</li>
<li>Interval Tree Clocks - for when processes come and go</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">GPS &amp; Atomic Clocks</h3><a id="user-content-gps--atomic-clocks" class="anchor" aria-label="Permalink: GPS &amp; Atomic Clocks" href="#gps--atomic-clocks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Much better than NTP
<ul dir="auto">
<li>Globally distributed total orders on the scale of milliseconds</li>
<li>Promote an asynchronous network to a semi-synchronous one</li>
<li>Unlocks more efficient algorithms</li>
</ul>
</li>
<li>Only people with this right now are Google
<ul dir="auto">
<li>Spanner: globally distributed strongly consistent transactions</li>
<li>And they're not sharing</li>
</ul>
</li>
<li>More expensive than you'd like
<ul dir="auto">
<li>Several hundred per GPS receiver</li>
<li>Atomic clocks for local corroboration: $$$$?</li>
<li>Probably want multiple types of GPS clock
<ul dir="auto">
<li><a href="https://rachelbythebay.com/w/2015/09/07/noleap/" rel="nofollow">https://rachelbythebay.com/w/2015/09/07/noleap/</a>
<ul dir="auto">
<li>Confusing vendor checkbox which applied UTC corrections to GPS time</li>
</ul>
</li>
</ul>
</li>
<li>I don't know who's doing it yet, but I'd bet datacenters in the
future will offer dedicated HW interfaces for bounded-accuracy time.</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Review</h2><a id="user-content-review" class="anchor" aria-label="Permalink: Review" href="#review"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">We've covered the fundamental primitives of distributed systems. Nodes
exchange messages through a network, and both nodes and networks can fail in
various ways. Protocols like TCP and UDP give us primitive channels for
processes to communicate, and we can order events using clocks. Now, we'll
discuss some high-level <em>properties</em> of distributed systems.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Availability</h2><a id="user-content-availability" class="anchor" aria-label="Permalink: Availability" href="#availability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Availability is basically the fraction of attempted operations which succeed.</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Total availability</h3><a id="user-content-total-availability" class="anchor" aria-label="Permalink: Total availability" href="#total-availability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Naive: every operation succeeds</li>
<li>In consistency lit: every operation on a non-failing node succeeds
<ul dir="auto">
<li>Nothing you can do about the failing nodes</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Sticky availability</h3><a id="user-content-sticky-availability" class="anchor" aria-label="Permalink: Sticky availability" href="#sticky-availability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Every operation against a non-failing node succeeds
<ul dir="auto">
<li>With the constraint that clients always talk to the same nodes</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">High availability</h3><a id="user-content-high-availability" class="anchor" aria-label="Permalink: High availability" href="#high-availability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Better than if the system <em>weren't</em> distributed.</li>
<li>e.g. tolerant of up to f failures, but no more</li>
<li>Maybe some operations fail</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Majority available</h3><a id="user-content-majority-available" class="anchor" aria-label="Permalink: Majority available" href="#majority-available"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Operations succeed <em>if</em> they occur on a node which can communicate
with a majority of the cluster</li>
<li>Operations against minority components may fail</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Quantifying availability</h3><a id="user-content-quantifying-availability" class="anchor" aria-label="Permalink: Quantifying availability" href="#quantifying-availability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>We talk a lot about "uptime"
<ul dir="auto">
<li>Are systems up if nobody uses them?</li>
<li>Is it worse to be down during peak hours?</li>
<li>Can measure "fraction of requests satisfied during a time window"</li>
<li>Then plot that fraction over windows at different times</li>
<li>Timescale affects reported uptime</li>
</ul>
</li>
<li>Apdex
<ul dir="auto">
<li>Not all successes are equal</li>
<li>Classify operations into "OK", "meh", and "awful"</li>
<li>Apdex = P(OK) + P(meh)/2</li>
<li>Again, can report on a yearly basis
<ul dir="auto">
<li>"We achieved 99.999 apdex for the year"</li>
</ul>
</li>
<li>And on finer timescales!
<ul dir="auto">
<li>"Apdex for the user service just dropped to 0.5; page ops!"</li>
</ul>
</li>
</ul>
</li>
<li>Ideally: integral of happiness delivered by your service?</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Consistency</h2><a id="user-content-consistency" class="anchor" aria-label="Permalink: Consistency" href="#consistency"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>A consistency model is the set of "safe" histories of events in the system</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Monotonic Reads</h3><a id="user-content-monotonic-reads" class="anchor" aria-label="Permalink: Monotonic Reads" href="#monotonic-reads"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Once I read a value, any subsequent read will return that state or later values</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Monotonic Writes</h3><a id="user-content-monotonic-writes" class="anchor" aria-label="Permalink: Monotonic Writes" href="#monotonic-writes"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>If I make a write, any subsequent writes I make will take place <em>after</em> the
first write</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Read Your Writes</h3><a id="user-content-read-your-writes" class="anchor" aria-label="Permalink: Read Your Writes" href="#read-your-writes"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Once I write a value, any subsequent read I perform will return that write
(or later values)</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Writes Follow Reads</h3><a id="user-content-writes-follow-reads" class="anchor" aria-label="Permalink: Writes Follow Reads" href="#writes-follow-reads"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Once I read a value, any subsequent write will take place after that read</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Causal consistency</h3><a id="user-content-causal-consistency" class="anchor" aria-label="Permalink: Causal consistency" href="#causal-consistency"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Suppose operations can be linked by a DAG of causal relationships
<ul dir="auto">
<li>A write that follows a read, for instance, is causally related
<ul dir="auto">
<li>Assuming the process didn't just throw away the read data</li>
</ul>
</li>
<li>Operations not linked in that DAG are <em>concurrent</em></li>
</ul>
</li>
<li>Constraint: before a process can execute an operation, all its precursors
must have executed on that node</li>
<li>Concurrent ops can be freely reordered</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Sequential consistency</h3><a id="user-content-sequential-consistency" class="anchor" aria-label="Permalink: Sequential consistency" href="#sequential-consistency"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Like causal consistency, constrains possible orders</li>
<li>All operations appear to execute atomically</li>
<li>Every process agrees on the order of operations
<ul dir="auto">
<li>Operations from a given process always occur in order</li>
<li>But nodes can lag behind</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Linearizability</h3><a id="user-content-linearizability" class="anchor" aria-label="Permalink: Linearizability" href="#linearizability"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>All operations appear to execute atomically</li>
<li>Every process agrees on the order of operations</li>
<li>Every operation appears to take place <em>between</em> its invocation and completion
times</li>
<li>Real-time, external constraints let us build very strong systems</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Transactional Models</h3><a id="user-content-transactional-models" class="anchor" aria-label="Permalink: Transactional Models" href="#transactional-models"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Serializable: all operations (transactions) appear to execute atomically
<ul dir="auto">
<li>In some order
<ul dir="auto">
<li>No constraints on what that order is</li>
<li>Perfectly okay to read from the past, for instance</li>
</ul>
</li>
</ul>
</li>
<li>Adya 1999: Weak Consistency: A Generalized Theory and Optimistic
Implementations for Distributed Transactions
<ul dir="auto">
<li>Objects have a total version order
<ul dir="auto">
<li>x0 &lt;&lt; x1 &lt;&lt; x2</li>
<li>This is an abstract requirement; DB may not know what the version order is</li>
</ul>
</li>
<li>Dependencies between transactions
<ul dir="auto">
<li>write-read (wr): T1 writes x1, T2 reads x1</li>
<li>write-write (ww): T1 writes x1, T2 writes x2</li>
<li>read-write (rw): T1 reads x1, T2 writes x2</li>
</ul>
</li>
<li>Phenomena are (mostly) cycles made of these edges
<ul dir="auto">
<li>G0 (write cycle): cycle in ww</li>
<li>G1a (aborted read): committed txn reads version written by aborted txn</li>
<li>G1b (intermediate read): read non-final write of x by some other txn</li>
<li>G1c (cyclic information flow): cycle in ww U wr</li>
<li>G2 (anti-dependency cycle): cycle in ww U wr U rw</li>
<li>G2-item: G2 without predicates</li>
<li>G-single: G2 with only one rw</li>
<li>etc. etc.</li>
</ul>
</li>
<li>Isolation levels prevent these phemomena
<ul dir="auto">
<li>Read Uncommitted: no G0</li>
<li>Read Committed: no G0, G1</li>
<li>Repeatable Read: no G0, G1, G2-item</li>
<li>SI: No g0, G1, G-single
<ul dir="auto">
<li>It's a little more subtle than this, but we generally don't have time
for G-nonadjacent</li>
</ul>
</li>
<li>Serializable: no G0, G1, G2</li>
</ul>
</li>
<li>Can augment graphs with process or realtime edges
<ul dir="auto">
<li>Strong X means "X, plus order of txns in real time"</li>
<li>Strong Session X means "X, plus order of txns in each session"</li>
<li>e.g. Strong Serializable: no cycle in ww U wr U rw U realtime</li>
</ul>
</li>
</ul>
</li>
<li>Alternate route: ANSI SQL's ACID isolation levels are weird
<ul dir="auto">
<li>Basically codified the effects of existing vendor implementations</li>
<li>Definitions in the spec are ambiguous</li>
<li>Each ANSI SQL isolation level prohibits a weird phenomenon</li>
<li>Read Uncommitted
<ul dir="auto">
<li>Prevents P0: <em>dirty writes</em>
<ul dir="auto">
<li>w1(x) ... w2(x)</li>
<li>Can't write over another transaction's data until it commits</li>
</ul>
</li>
<li>Can read data while a transaction is still modifying it</li>
<li>Can read data that will be rolled back</li>
</ul>
</li>
<li>Read Committed
<ul dir="auto">
<li>Prevents P1: <em>dirty reads</em>
<ul dir="auto">
<li>w1(x) ... r2(x)</li>
<li>Can't read a transaction's uncommitted values</li>
</ul>
</li>
</ul>
</li>
<li>Repeatable Read
<ul dir="auto">
<li>Prevents P2: <em>fuzzy reads</em>
<ul dir="auto">
<li>r1(x) ... w2(x)</li>
<li>Once a transaction reads a value, it won't change until the transaction
commits</li>
</ul>
</li>
</ul>
</li>
<li>Serializable
<ul dir="auto">
<li>Prevents P3: <em>phantoms</em>
<ul dir="auto">
<li>Given some predicate P</li>
<li>r1(P) ... w2(y in P)</li>
<li>Once a transaction reads a set of elements satisfying a query, that
set won't change until the transaction commits</li>
<li>Not just values, but <em>which values even would have participated</em>.</li>
</ul>
</li>
</ul>
</li>
<li>Cursor Stability
<ul dir="auto">
<li>Transactions have a set of cursors
<ul dir="auto">
<li>A cursor refers to an object being accessed by the transaction</li>
</ul>
</li>
<li>Read locks are held until cursor is removed, or commit
<ul dir="auto">
<li>At commit time, cursor is upgraded to a writelock</li>
</ul>
</li>
<li>Prevents lost-update</li>
</ul>
</li>
<li>Snapshot Isolation
<ul dir="auto">
<li>Transactions always read from a snapshot of committed data, taken before
the transaction begins</li>
<li>Commit can only occur if no other committed transaction with an
overlapping [start..commit] interval has written to any of the objects
<em>we</em> wrote
<ul dir="auto">
<li>First-committer-wins</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Does any of this actually matter?</h3><a id="user-content-does-any-of-this-actually-matter" class="anchor" aria-label="Permalink: Does any of this actually matter?" href="#does-any-of-this-actually-matter"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Real world just isn't that concurrent</li>
<li>Plenty of companies get by on Read Committed</li>
<li>But <em>malicious</em> attackers can <em>induce</em> concurrency
<ul dir="auto">
<li>Flexcoin
<ul dir="auto">
<li>Bitcoin exchange which allowed users to create money by shuffling between accounts</li>
<li>Attacked in 2014, 365,000 GBP stolen</li>
<li>Exchange collapsed altogether</li>
</ul>
</li>
<li>Poloniex
<ul dir="auto">
<li>Concurrent withdrawals were improperly isolated, allowing users to overspend</li>
<li>Safety audits didn't notice negative balances</li>
<li>12.3% of exchange funds stolen; loss spread among users</li>
</ul>
</li>
<li><a href="http://www.bailis.org/papers/acidrain-sigmod2017.pdf" rel="nofollow">Warszawski &amp; Bailis 2017: ACIDRain</a>
<ul dir="auto">
<li>Automated identification of consistency violation in web apps</li>
<li>e.g. Buy one gift card, then spend it an unlimited number of times</li>
<li>e.g. Buy a pen, add a laptop to cart during checkout, get a free laptop</li>
<li>Vulnerabilities found in over 50% of all eCommerce web sites
<ul dir="auto">
<li>Weak DB isolation defaults</li>
<li>Improper use of transactional scope</li>
<li>Failing to use any transactions whatsoever</li>
</ul>
</li>
</ul>
</li>
<li><a href="https://chadscira.com/post/5fa269d46142ac544e013d6e/DISCLOSURE-Unlimited-Chase-Ultimate-Rewards-Points" rel="nofollow">Chase Bank's credit card rewards system</a>
<ul dir="auto">
<li>Concurrent transfers between balances allowed the creation of $70,000 USD
in travel vouchers.</li>
<li>Redeemable for cash!</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Tradeoffs</h2><a id="user-content-tradeoffs" class="anchor" aria-label="Permalink: Tradeoffs" href="#tradeoffs"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Ideally, we want total availability and linearizability</li>
<li>Consistency requires coordination
<ul dir="auto">
<li>If every order is allowed, we don't need to do any work!</li>
<li>If we want to disallow some orders of events, we have to exchange messages</li>
</ul>
</li>
<li>Coordinating comes (generally) with costs
<ul dir="auto">
<li>More consistency is slower</li>
<li>More consistency is more intuitive</li>
<li>More consistency is less available</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Availability and Consistency</h3><a id="user-content-availability-and-consistency" class="anchor" aria-label="Permalink: Availability and Consistency" href="#availability-and-consistency"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>CAP Theorem: Linearizability OR total availability</li>
<li>But wait, there's more!
<ul dir="auto">
<li>Bailis 2014: Highly Available Transactions: Virtues and Limitations</li>
<li>Other theorems disallow totally or sticky available...
<ul dir="auto">
<li>Strong serializable</li>
<li>Serializable</li>
<li>Repeatable Read</li>
<li>Cursor Stability</li>
<li>Snapshot Isolation</li>
</ul>
</li>
<li>You can have <em>sticky</em> available...
<ul dir="auto">
<li>Causal</li>
<li>PRAM</li>
<li>Read Your Writes</li>
</ul>
</li>
<li>You can have <em>totally</em> available...
<ul dir="auto">
<li>Read Uncommitted</li>
<li>Read Committed</li>
<li>Monotonic Atomic View</li>
<li>Writes Follow Reads</li>
<li>Monotonic Reads</li>
<li>Monotonic Writes</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Harvest and Yield</h3><a id="user-content-harvest-and-yield" class="anchor" aria-label="Permalink: Harvest and Yield" href="#harvest-and-yield"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Fox &amp; Brewer, 1999: Harvest, Yield, and Scalable Tolerant Systems
<ul dir="auto">
<li>Yield: probability of completing a request</li>
<li>Harvest: fraction of data reflected in the response</li>
<li>Examples
<ul dir="auto">
<li>Node faults in a search engine can cause some results to go missing</li>
<li>Updates may be reflected on some nodes but not others
<ul dir="auto">
<li>Consider an AP system split by a partition</li>
<li>You can write data that some people can't read</li>
</ul>
</li>
<li>Streaming video degrades to preserve low latency</li>
</ul>
</li>
<li>This is not an excuse to violate your safety invariants
<ul dir="auto">
<li>Just helps you quantify how much you can <em>exceed</em> safety invariants</li>
<li>e.g. "99% of the time, you can read 90% of your prior writes"</li>
</ul>
</li>
<li>Strongly dependent on workload, HW, topology, etc</li>
<li>Can tune harvest vs yield on a per-request basis</li>
<li>"As much as possible in 10ms, please"</li>
<li>"I need everything, and I understand you might not be able to answer"</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Hybrid systems</h3><a id="user-content-hybrid-systems" class="anchor" aria-label="Permalink: Hybrid systems" href="#hybrid-systems"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>So, you've got a spectrum of choices!
<ul dir="auto">
<li>Chances are different parts of your infrastructure have different needs</li>
<li>Pick the weakest model that meets your constraints
<ul dir="auto">
<li>But consider probabilistic bounds; visibility lag might be prohibitive</li>
<li>See Probabilistically Bounded Staleness in Dynamo Quorums</li>
</ul>
</li>
</ul>
</li>
<li>Not all data is equal
<ul dir="auto">
<li>Big data is usually less important</li>
<li>Small data is usually critical</li>
<li>Linearizable user ops, causally consistent social feeds</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Review</h3><a id="user-content-review-1" class="anchor" aria-label="Permalink: Review" href="#review-1"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Availability is a measure of how often operations succeed. Consistency models
are the rules that govern what operations can happen and when. Stronger
consistency models generally come at the cost of performance and availability.
Next, we'll talk about different ways to build systems, from weak to strong
consistency.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Avoid Consensus Wherever Possible</h2><a id="user-content-avoid-consensus-wherever-possible" class="anchor" aria-label="Permalink: Avoid Consensus Wherever Possible" href="#avoid-consensus-wherever-possible"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">CALM conjecture</h3><a id="user-content-calm-conjecture" class="anchor" aria-label="Permalink: CALM conjecture" href="#calm-conjecture"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Consistency As Logical Monotonicity
<ul dir="auto">
<li>If you can prove a system is logically monotonic, it is coordination free</li>
<li>What the heck is "coordination"</li>
<li>For that matter, what's "monotonic"?</li>
</ul>
</li>
<li>Monotonicity, informally, is retraction-free
<ul dir="auto">
<li>Deductions from partial information are never invalidated by new information</li>
<li>Both relational algebra and Datalog without negation are monotone</li>
</ul>
</li>
<li>Ameloot, et al, 2011: Relational transducers for declarative networking
<ul dir="auto">
<li>Theorem which shows coordination-free networks of processes unaware of the
network extent can compute only monotone queries in Datalog
<ul dir="auto">
<li>This is not an easy read</li>
</ul>
</li>
<li>"Coordination-free" doesn't mean no communication
<ul dir="auto">
<li>Algo succeeds even in face of arbitrary horizontal partitions</li>
</ul>
</li>
</ul>
</li>
<li>In very loose practical terms
<ul dir="auto">
<li>Try to phrase your problem such that you only <em>add</em> new facts to the system</li>
<li>When you compute a new fact based on what's currently known, can you ensure
that fact will never be retracted?</li>
<li>Consider special "sealing facts" that mark a block of facts as complete</li>
<li>These "grow-only" algorithms are usually easier to implement</li>
<li>Likely tradeoff: incomplete reads</li>
</ul>
</li>
<li>Bloom language
<ul dir="auto">
<li>Unordered programming with flow analysis</li>
<li>Can tell you where coordination <em>would</em> be required</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Gossip</h3><a id="user-content-gossip" class="anchor" aria-label="Permalink: Gossip" href="#gossip"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Message broadcast system</li>
<li>Useful for cluster management, service discovery, health, sensors, CDNs, etc</li>
<li>Generally weak consistency / high availability</li>
<li>Global broadcast
<ul dir="auto">
<li>Send a message to every other node</li>
<li>O(nodes)</li>
</ul>
</li>
<li>Mesh networks
<ul dir="auto">
<li>Epidemic models</li>
<li>Relay to your neighbors</li>
<li>Propagation times on the order of max-free-path</li>
</ul>
</li>
<li>Spanning trees
<ul dir="auto">
<li>Instead of a mesh, use a tree</li>
<li>Hop up to a connector node which relays to other connector nodes</li>
<li>Reduces superfluous messages</li>
<li>Reduces latency</li>
<li>Plumtree (Leit ̃ao, Pereira, &amp; Rodrigues, 2007: Epidemic Broadcast Trees)</li>
</ul>
</li>
<li>Push-Sum
<ul dir="auto">
<li>Kempe, Dobra, &amp; Gehrke - Gossip-Based Computation of Aggregate Information</li>
<li>Sum inputs from everyone you've received data from</li>
<li>Broadcast that to a random peer</li>
<li>Extensions for minima, maxima, means</li>
<li>Helpful for live metrics, rate limiting, routing, identifying cluster
hotspots</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">CRDTs</h3><a id="user-content-crdts" class="anchor" aria-label="Permalink: CRDTs" href="#crdts"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Order-free datatypes that converge
<ul dir="auto">
<li>Counters, sets, maps, etc</li>
</ul>
</li>
<li>Tolerate dupes, delays, and reorders</li>
<li>Unlike sequentially consistent systems, no "single source of truth"</li>
<li>But unlike naive eventually consistent systems, never <em>lose</em> information
<ul dir="auto">
<li>Unless you explicitly make them lose information</li>
<li>We call this property "coalescence"</li>
</ul>
</li>
<li>Works well in highly-available systems
<ul dir="auto">
<li>Web/mobile clients</li>
<li>Dynamo</li>
<li>Gossip</li>
</ul>
</li>
<li>INRIA: Shapiro, Preguiça, Baquero, Zawirski, 2011: "A comprehensive study of
Convergent and Commutative Replicated Data Types"
<ul dir="auto">
<li>Composed of a data type X and a merge function m, which is:
<ul dir="auto">
<li>Associative: m(x1, m(x2, x3)) = m(m(x1, x2), x3)</li>
<li>Commutative: m(x1, x2) = m(x2, x1)</li>
<li>Idempotent: m(x1, x1) = m(x1)</li>
</ul>
</li>
</ul>
</li>
<li>Easy to build. Easy to reason about. Gets rid of all kinds of headaches.
<ul dir="auto">
<li>Did communication fail? Just retry! It'll converge!</li>
<li>Did messages arrive out of order? It's fine!</li>
<li>How do I synchronize two replicas? Just merge!</li>
</ul>
</li>
<li>Downsides
<ul dir="auto">
<li>Some algorithms <em>need</em> order and can't be expressed with CRDTs</li>
<li>Reads may be arbitrarily stale</li>
<li>Higher space costs</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">HATs</h3><a id="user-content-hats" class="anchor" aria-label="Permalink: HATs" href="#hats"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Bailis, Davidson, Fekete, et al, 2013: "Highly Available Transactions,
Virtues and Limitations"
<ul dir="auto">
<li>Guaranteed responses from any replica</li>
<li>Low latency (1-3 orders of magnitude faster than serializable protocols!)</li>
<li>Read Committed</li>
<li>Monotonic Atomic View</li>
<li>Excellent for commutative/monotonic systems</li>
<li>Foreign key constraints for multi-item updates</li>
<li>Limited uniqueness constraints</li>
<li>Can ensure convergence given arbitrary finite delay ("eventual consistency")</li>
<li>Good candidates for geographically distributed systems</li>
<li>Probably best in concert with stronger transactional systems</li>
<li>See also: COPS, Swift, Eiger, Calvin, etc</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Fine, We Need Consensus, What Now?</h2><a id="user-content-fine-we-need-consensus-what-now" class="anchor" aria-label="Permalink: Fine, We Need Consensus, What Now?" href="#fine-we-need-consensus-what-now"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>
<p dir="auto">The consensus problem:</p>
<ul dir="auto">
<li>Three process types
<ul dir="auto">
<li>Proposers: propose values</li>
<li>Acceptors: choose a value</li>
<li>Learners: read the chosen value</li>
</ul>
</li>
<li>Classes of acceptors
<ul dir="auto">
<li>N acceptors total</li>
<li>F acceptors allowed to fail</li>
<li>M malicious acceptors</li>
</ul>
</li>
<li>Three invariants:
<ul dir="auto">
<li>Nontriviality: Only values proposed can be learned</li>
<li>Safety: At most one value can be learned</li>
<li>Liveness: If a proposer p, a learner l, and a set of N-F acceptors are
non-faulty and can communicate with each other, and if p proposes a
value, l will eventually learn a value.</li>
</ul>
</li>
</ul>
</li>
<li>
<p dir="auto">Whole classes of systems are <em>equivalent</em> to the consensus problem</p>
<ul dir="auto">
<li>So any proofs we have here apply to those systems too</li>
<li>Lock services</li>
<li>Ordered logs</li>
<li>Replicated state machines</li>
</ul>
</li>
<li>
<p dir="auto">FLP tells us consensus is impossible in asynchronous networks</p>
<ul dir="auto">
<li>Kill a process at the right time and you can break <em>any</em> consensus algo</li>
<li>True but not as bad as you might think</li>
<li>Realistically, networks work <em>often enough</em> to reach consensus</li>
<li>Moreover, FLP assumes deterministic processes
<ul dir="auto">
<li>Real computers <em>aren't</em> deterministic</li>
<li>Ben-Or 1983: "Another Advantage of free choice"
<ul dir="auto">
<li>Nondeterministic algorithms <em>can</em> achieve consensus</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li>
<p dir="auto">Lamport 2002: tight bounds for asynchronous consensus</p>
<ul dir="auto">
<li>With at least two proposers, or one malicious proposer, N &gt; 2F + M
<ul dir="auto">
<li>"Need a majority"</li>
</ul>
</li>
<li>With at least 2 proposers, or one malicious proposer, it takes at least 2
message delays to learn a proposal.</li>
</ul>
</li>
<li>
<p dir="auto">This is a pragmatically achievable bound</p>
<ul dir="auto">
<li>In stable clusters, you can get away with only a single round-trip to a
majority of nodes.</li>
<li>More during cluster transitions.</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Paxos</h3><a id="user-content-paxos" class="anchor" aria-label="Permalink: Paxos" href="#paxos"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Paxos is the Gold Standard of consensus algorithms
<ul dir="auto">
<li>Lamport 1989 - The Part Time Parliament
<ul dir="auto">
<li>Written as a description of an imaginary Greek democracy</li>
</ul>
</li>
<li>Lamport 2001 - Paxos Made Simple
<ul dir="auto">
<li>"The Paxos algorithm for implementing a fault-tolerant distributed system
has been regarded as difficult to understand, perhaps because the
original presentation was Greek to many readers [5]. In fact, it is among the
simplest and most obvious of distributed algorithms... The last section
explains the complete Paxos algorithm, which is obtained by the straightforward
application of consensus to the state machine approach for building a
distributed system—an approach that should be well-known, since it is the
subject of what is probably the most often-cited article on the theory of
distributed systems [4]."</li>
</ul>
</li>
<li>Google 2007 - Paxos Made Live
<ul dir="auto">
<li>Notes from productionizing Chubby, Google's lock service</li>
</ul>
</li>
<li>Van Renesse 2011 - Paxos Made Moderately Complex
<ul dir="auto">
<li>Turns out you gotta optimize</li>
<li>Also pseudocode would help</li>
<li>A page of pseudocode -&gt; several thousand lines of C++</li>
</ul>
</li>
</ul>
</li>
<li>Provides consensus on independent proposals</li>
<li>Typically deployed in majority quorums, 5 or 7 nodes</li>
<li>Several optimizations
<ul dir="auto">
<li>Multi-Paxos</li>
<li>Fast Paxos</li>
<li>Generalized Paxos</li>
<li>It's not always clear which of these optimizations to use, and which
can be safely combined</li>
<li>Each implementation uses a slightly different flavor</li>
<li>Paxos is really more of a <em>family</em> of algorithms than a well-described
single entity</li>
</ul>
</li>
<li>Used in a variety of production systems
<ul dir="auto">
<li>Chubby</li>
<li>Cassandra</li>
<li>Riak</li>
<li>FoundationDB</li>
<li>WANdisco SVN servers</li>
</ul>
</li>
<li>New research: Paxos quorums need not be majority: can optimize for fast phase-2 quorums <a href="https://arxiv.org/abs/1608.06696" rel="nofollow">Howard, Malkhi, and Spiegelman</a>.
<ul dir="auto">
<li>We're not sure how to USE this yet</li>
<li>Durability still requires distribution</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">ZAB</h3><a id="user-content-zab" class="anchor" aria-label="Permalink: ZAB" href="#zab"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>ZAB is the Zookeeper Atomic Broadcast protocol</li>
<li>Junqueira, Reed, and Serafini 2011 - Zab: High-performance broadcast for
primary-backup systems</li>
<li>Differs from Paxos</li>
<li>Provides sequential consistency (linearizable writes, lagging ordered reads)
<ul dir="auto">
<li>Useful because ZK clients typically want fast local reads</li>
<li>But there's also a SYNC command that guarantees real-time visibility</li>
<li>(SYNC + op) allows linearizable reads as well</li>
</ul>
</li>
<li>Again, majority quorum, 5 or 7 nodes</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Humming Consensus</h3><a id="user-content-humming-consensus" class="anchor" aria-label="Permalink: Humming Consensus" href="#humming-consensus"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Metadata store for managing distributed system reconfiguration</li>
<li>Looks a little like CORFU's replicated log</li>
<li>See also: chain replication</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Viewstamped Replication</h3><a id="user-content-viewstamped-replication" class="anchor" aria-label="Permalink: Viewstamped Replication" href="#viewstamped-replication"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Presented as a replication protocol, but also a consensus algorithm</li>
<li>Transaction processing plus a view change algorithm</li>
<li>Majority-known values are guaranteed to survive into the future</li>
<li>I'm not aware of any production systems, but I'm sure they're out there</li>
<li>Along with Paxos, inspired Raft in some ways</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Raft</h3><a id="user-content-raft" class="anchor" aria-label="Permalink: Raft" href="#raft"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Ongaro &amp; Ousterhout 2014 - In Search of an Understandable Consensus Algorithm</li>
<li>Lamport says it's easy, but we still have trouble grokking Paxos
<ul dir="auto">
<li>What if there were a consensus algorithm we could actually understand?</li>
</ul>
</li>
<li>Paxos approaches independent decisions when what we <em>want</em> is state machines
<ul dir="auto">
<li>Maintains a replicated <em>log</em> of state machine transitions instead</li>
</ul>
</li>
<li>Also builds in cluster membership transitions, which is <em>key</em> for real systems</li>
<li>Very new, but we have a Coq proof of the core algorithm</li>
<li>Can be used to write arbitrary sequential or linearizable state machines
<ul dir="auto">
<li>RethinkDB</li>
<li>etcd</li>
<li>Consul</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">What About Transactions?</h2><a id="user-content-what-about-transactions" class="anchor" aria-label="Permalink: What About Transactions?" href="#what-about-transactions"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Iterated consensus gives us agreement on a single total order of operations</li>
<li>Unnecessary blocking betwixt transactions which <em>could</em> execute independently</li>
<li>How do we improve performance?</li>
<li><a href="https://aphyr.com/media/talks/2019/distributed-transaction-architectures.pdf" rel="nofollow">Distributed Transaction Architectures</a></li>
<li>Single-writer
<ul dir="auto">
<li>All updates go through a single queue, readers execute on snapshots</li>
<li>Usually involves some kind of persistent data structure</li>
<li>Serializable to strict-1SR</li>
<li>See <a href="https://docs.datomic.com/on-prem/architecture.html" rel="nofollow">Datomic</a></li>
</ul>
</li>
<li>OK but multiple writers?
<ul dir="auto">
<li>In general, have several shards, each running a consensus-backed FSM</li>
<li>Some kind of protocol for cross-shard txns</li>
</ul>
</li>
<li>Independent shards
<ul dir="auto">
<li>A sort of halfway-step to general-purpose transactions</li>
<li>Disallow cross-shard txns</li>
<li>Just run a bunch of independent consensus FSMs</li>
<li>Can add a single global consensus group for cross-shard transactions
<ul dir="auto">
<li>Limited throughput though!</li>
</ul>
</li>
<li>See <a href="https://docs.voltdb.com/UsingVoltDB/IntroHowVoltDBWorks.php" rel="nofollow">VoltDB</a></li>
</ul>
</li>
<li><a href="https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf" rel="nofollow">Percolator</a>
<ul dir="auto">
<li>Snapshot isolation over linearizable shards</li>
<li>Time Stamp Oracle assigns sequential txn timestamps (using consensus)</li>
<li>Read timestamp, read from leaders, prewrite, commit timestamp, commit, finalize</li>
<li>14 network hops, potentially all cross-DC</li>
<li>See <a href="https://tikv.org/deep-dive/distributed-transaction/percolator/" rel="nofollow">TiDB</a></li>
</ul>
</li>
<li><a href="https://static.googleusercontent.com/media/research.google.com/en//archive/spanner-osdi2012.pdf" rel="nofollow">Spanner</a>
<ul dir="auto">
<li>"External consistency" (strict-1SR?)</li>
<li>Speed up timestamp assignment by using GPS+Atomic clocks</li>
<li>Basically 2PC over Paxos groups
<ul dir="auto">
<li>Locks on Paxos leaders</li>
<li>Pick one Paxos group to serve as the commit record for entire txn</li>
</ul>
</li>
<li>Fixed latency floor to ensure timestamp monotonicity</li>
<li>See <a href="https://blog.yugabyte.com/distributed-postgresql-on-a-google-spanner-architecture-storage-layer/" rel="nofollow">Yugabyte DB</a></li>
<li>See <a href="https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/" rel="nofollow">CockroachDB</a></li>
</ul>
</li>
<li><a href="http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf" rel="nofollow">Calvin</a>
<ul dir="auto">
<li>Order transactions in a log using consensus</li>
<li>Shard log for arbitrarily high throughput</li>
<li>Periodically seal log windows and apply transactions to shards</li>
<li>Application requires no communication!</li>
<li>Strict-1SR</li>
<li>1 inter-DC round trip, more hops for local comms</li>
<li>Scalable throughput</li>
<li>Minimum latency floor</li>
<li>Txns must be pure, expressed up-front
<ul dir="auto">
<li>Could be made interactive with extensions to protocol</li>
</ul>
</li>
<li>See <a href="https://fauna.com/" rel="nofollow">Fauna</a></li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Review</h2><a id="user-content-review-2" class="anchor" aria-label="Permalink: Review" href="#review-2"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Systems which only add facts, not retract them, require less coordination to
build. We can use gossip systems to broadcast messages to other processes,
CRDTs to merge updates from our peers, and HATs for weakly isolated
transactions. Serializability and linearizability require <em>consensus</em>, which we
can obtain through Paxos, ZAB, VR, or Raft. Now, we'll talk about different
<em>scales</em> of distributed systems.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Characteristic latencies</h2><a id="user-content-characteristic-latencies" class="anchor" aria-label="Permalink: Characteristic latencies" href="#characteristic-latencies"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Latency is <em>never</em> zero
<ul dir="auto">
<li>Bandwidth goes up and up but we're bumping up against the physical limits
of light and electrons</li>
<li>Latency budget shapes your system design
<ul dir="auto">
<li>How many network calls can you afford?</li>
</ul>
</li>
</ul>
</li>
<li>Different kinds of systems have different definitions of "slow"
<ul dir="auto">
<li>Different goals</li>
<li>Different algorithms</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Multicore systems</h3><a id="user-content-multicore-systems" class="anchor" aria-label="Permalink: Multicore systems" href="#multicore-systems"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Multicore (and especially NUMA) architectures are sort of like a distributed system
<ul dir="auto">
<li>Nodes don't fail pathologically, but message exchange is slow!</li>
<li>Synchronous network provided by a bus (e.g. Intel QPI)</li>
<li>Whole complicated set of protocols in HW &amp; microcode to make memory look
sane</li>
<li>Non-temporal store instructions (e.g. MOVNTI)</li>
</ul>
</li>
<li>They provide abstractions to hide that distribution
<ul dir="auto">
<li>MFENCE/SFENCE/LFENCE
<ul dir="auto">
<li>Introduce a serialization point against load/store instructions</li>
<li>Characteristic latencies: ~100 cycles / ~30 ns
<ul dir="auto">
<li>Really depends on HW, caches, instructions, etc</li>
</ul>
</li>
</ul>
</li>
<li>CMPXCHG Compare-and-Swap (sequentially consistent modification of memory)</li>
<li>LOCK
<ul dir="auto">
<li>Lock the full memory subsystem across cores!</li>
</ul>
</li>
</ul>
</li>
<li>But those abstractions come with costs
<ul dir="auto">
<li>Hardware lock elision may help but is nascent</li>
<li>Blog: Mechanical Sympathy</li>
<li>Avoid coordination between cores wherever possible</li>
<li>Context switches (process or thread!) can be expensive</li>
<li>Processor pinning can really improve things</li>
<li>When writing multithreaded programs, try to divide your work into
independent chunks
<ul dir="auto">
<li>Try to align memory barriers to work unit boundaries</li>
<li>Allows the processor to cheat as much as possible within a work unit</li>
</ul>
</li>
<li>See Danica Porobic, 2016: <a href="https://infoscience.epfl.ch/record/219117/files/EPFL_TH7023.pdf" rel="nofollow">High Performance Transaction Processing on Non-Uniform Hardware Topologies</a></li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Local networks</h3><a id="user-content-local-networks" class="anchor" aria-label="Permalink: Local networks" href="#local-networks"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>You'll often deploy replicated systems across something like an ethernet LAN</li>
<li>Message latencies can be as low as 100 micros
<ul dir="auto">
<li>But across any sizable network (EC2), expect low millis</li>
<li>Sometimes, packets could be delayed by <em>five minutes</em></li>
<li>Plan for this</li>
</ul>
</li>
<li>Network is within an order of mag compared to uncached disk seeks
<ul dir="auto">
<li>Or faster, in EC2
<ul dir="auto">
<li>EC2 disk latencies can routinely hit 20ms
<ul dir="auto">
<li>200ms?
<ul dir="auto">
<li><em>20,000</em> ms???
<ul dir="auto">
<li>Because EBS is actually other computers</li>
<li>LMAO if you think anything in EC2 is real
<ul dir="auto">
<li>Wait, <em>real disks do this too</em>?
<ul dir="auto">
<li>What even are IO schedulers?</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li>But network is waaaay slower than memory/computation
<ul dir="auto">
<li>If your aim is <em>throughput</em>, work units should probably take longer than a
millisecond</li>
<li>But there are other reasons to distribute
<ul dir="auto">
<li>Sharding resources</li>
<li>Isolating failures</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Geographic replication</h3><a id="user-content-geographic-replication" class="anchor" aria-label="Permalink: Geographic replication" href="#geographic-replication"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>You deploy worldwide for two reasons
<ul dir="auto">
<li>End-user latency
<ul dir="auto">
<li>Humans can detect ~10ms lag, will tolerate ~100ms
<ul dir="auto">
<li>SF--Denver: 50ms</li>
<li>SF--Tokyo: 100 ms</li>
<li>SF--Madrid: 200 ms</li>
</ul>
</li>
<li>Only way to beat the speed of light: move the service closer</li>
</ul>
</li>
<li>Disaster recovery
<ul dir="auto">
<li>Datacenter power is good but not perfect</li>
<li>Hurricanes are a thing</li>
<li>Entire Amazon regions can and will fail
<ul dir="auto">
<li>Yes, regions, not AZs</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li>Minimum of 1 round-trip for consensus
<ul dir="auto">
<li>Maybe as bad as 4 rounds
<ul dir="auto">
<li>Maybe 4 rounds all the time if you have a bad Paxos impl (e.g. Cassandra)</li>
</ul>
</li>
<li>So if you do Paxos between datacenters, be ready for that cost!</li>
<li>Because the minimum latencies are higher than users will tolerate
<ul dir="auto">
<li>Cache cache cache</li>
<li>Queue writes and relay asynchronously</li>
<li>Consider reduced consistency guarantees in exchange for lower latency</li>
<li>CRDTs can always give you safe local writes</li>
<li>Causal consistency and HATs can be good calls here</li>
</ul>
</li>
</ul>
</li>
<li>What about strongly consistent stuff?
<ul dir="auto">
<li>Chances are a geographically distributed service has natural planes of
cleavage
<ul dir="auto">
<li>EU users live on EU servers; US users live on US servers</li>
<li>Use consensus to migrate users between datacenters</li>
</ul>
</li>
<li>Pin/proxy updates to home datacenter
<ul dir="auto">
<li>Which is hopefully the closest datacenter!</li>
<li>But maybe not! I believe Facebook still pushes all writes through 1 DC!</li>
</ul>
</li>
<li>Where sequential consistency is OK, cache reads locally!
<ul dir="auto">
<li>You probably leverage caching in a single DC already</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Review</h3><a id="user-content-review-3" class="anchor" aria-label="Permalink: Review" href="#review-3"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">We discussed three characteristic scales for distributed systems: multicore
processors coupled with a synchronous network, computers linked by a LAN, and
datacenters linked by the internet or dedicated fiber. CPU consequences are
largely performance concerns: knowing how to minimize coordination. On LANs,
latencies are short enough for many network hops before users take notice. In
geographically replicated systems, high latencies drive eventually consistent
and datacenter-pinned solutions.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Common distributed systems</h2><a id="user-content-common-distributed-systems" class="anchor" aria-label="Permalink: Common distributed systems" href="#common-distributed-systems"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Outsourced heaps</h3><a id="user-content-outsourced-heaps" class="anchor" aria-label="Permalink: Outsourced heaps" href="#outsourced-heaps"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Redis, memcached, ...</li>
<li>Data fits in memory, complex data structures</li>
<li>Useful when your language's built-in data structures are slow/awful</li>
<li>Excellent as a cache</li>
<li>Or as a quick-and-dirty scratchpad for shared state between platforms</li>
<li>Not particularly safe</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">KV stores</h3><a id="user-content-kv-stores" class="anchor" aria-label="Permalink: KV stores" href="#kv-stores"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Riak, Couch, Mongo, Cassandra, RethinkDB, HDFS, ...</li>
<li>Often 1,2,3 dimensions of keys</li>
<li>O(1) access, sometimes O(range) range scans by ID</li>
<li>No strong relationships between values</li>
<li>Objects may be opaque or structured</li>
<li>Large data sets</li>
<li>Often linear scalability</li>
<li>Often no transactions</li>
<li>Range of consistency models--often optional linearizable/sequential ops.</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">SQL databases</h3><a id="user-content-sql-databases" class="anchor" aria-label="Permalink: SQL databases" href="#sql-databases"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Postgres, MySQL, Percona XtraDB, Oracle, MSSQL, VoltDB, CockroachDB, ...</li>
<li>Defined by relational algebra: restrictions of products of records, etc</li>
<li>Moderate sized data sets</li>
<li>Almost always include multi-record transactions</li>
<li>Relations and transactions require coordination, which reduces scalability</li>
<li>Many systems are primary-secondary failover</li>
<li>Access cost varies depending on indexes</li>
<li>Typically strong consistency (SI, serializable, strict serializable)</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Search</h3><a id="user-content-search" class="anchor" aria-label="Permalink: Search" href="#search"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Elasticsearch, SolrCloud, ...</li>
<li>Documents referenced by indices</li>
<li>Moderate-to-large data sets</li>
<li>Usually O(1) document access, log-ish search</li>
<li>Good scalability</li>
<li>Typically weak consistency</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Coordination services</h3><a id="user-content-coordination-services" class="anchor" aria-label="Permalink: Coordination services" href="#coordination-services"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Zookeeper, etcd, Consul, ...</li>
<li>Typically strong (sequential or linearizable) consistency</li>
<li>Small data sets</li>
<li>Useful as a coordination primitive for stateless services</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Streaming systems</h3><a id="user-content-streaming-systems" class="anchor" aria-label="Permalink: Streaming systems" href="#streaming-systems"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Storm, Spark...</li>
<li>Usually custom-designed, or toolkits to build your own.</li>
<li>Typically small in-memory data volume</li>
<li>Low latencies</li>
<li>High throughput</li>
<li>Weak consistency</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Distributed queues</h3><a id="user-content-distributed-queues" class="anchor" aria-label="Permalink: Distributed queues" href="#distributed-queues"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Kafka, Kestrel, Rabbit, IronMQ, ActiveMQ, HornetQ, Beanstalk, SQS, Celery, ...</li>
<li>Journals work to disk on multiple nodes for redundancy</li>
<li>Useful when you need to acknowledge work now, and actually do it later</li>
<li>Send data reliably between stateless services</li>
<li>The <em>only</em> one I know that won't lose data in a partition is Kafka
<ul dir="auto">
<li>Maybe SQS?</li>
</ul>
</li>
<li>Queues do not improve end-to-end latency
<ul dir="auto">
<li>Always faster to do the work immediately</li>
</ul>
</li>
<li>Queues do not improve mean throughput
<ul dir="auto">
<li>Mean throughput limited by consumers</li>
</ul>
</li>
<li>Queues do not provide total event ordering when consumers are concurrent
<ul dir="auto">
<li>Your consumers are almost definitely concurrent</li>
</ul>
</li>
<li>Likewise, queues don't guarantee event order with async consumers
<ul dir="auto">
<li>Because consumer side effects could take place out of order</li>
<li>So, don't rely on order</li>
</ul>
</li>
<li>Queues can offer at-most-once or at-least-once delivery
<ul dir="auto">
<li>Anyone claiming otherwise is trying to sell you something</li>
<li>Recovering exactly-once delivery requires careful control of side effects</li>
<li>Make your queued operations idempotent</li>
</ul>
</li>
<li>Queues do improve burst throughput
<ul dir="auto">
<li>Smooth out load spikes</li>
</ul>
</li>
<li>Distributed queues also improve fault tolerance (if they don't lose data)
<ul dir="auto">
<li>If you don't need the fault-tolerance or large buffering, just use TCP</li>
<li>Lots of people use a queue with six disk writes and fifteen network hops
where a single socket write() could have sufficed</li>
</ul>
</li>
<li>Queues can get you out of a bind when you've chosen a poor runtime</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Review</h3><a id="user-content-review-4" class="anchor" aria-label="Permalink: Review" href="#review-4"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">We use data structure stores as outsourced heaps: they're the duct tape of
distributed systems. KV stores and relational databases are commonly deployed
as systems of record; KV stores use independent keys and are not well-suited to
relational data, but offer improved scalability and partial failure vs SQL
stores, which offer rich queries and strong transactional guarantees.
Distributed search and coordination services round out our basic toolkit for
building applications. Streaming systems are applied for continuous,
low-latency processing of datasets, and tend to look more like frameworks than
databases. Their dual, distributed queues, focus on the <em>messages</em> rather
than the <em>transformations</em>.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">A Pattern Language</h2><a id="user-content-a-pattern-language" class="anchor" aria-label="Permalink: A Pattern Language" href="#a-pattern-language"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>General recommendations for building distributed systems
<ul dir="auto">
<li>Hard-won experience</li>
<li>Repeating what other experts tell me
<ul dir="auto">
<li>Over beers</li>
</ul>
</li>
<li>Hearsay</li>
<li>Oversimplifications</li>
<li>Cargo-culting</li>
<li>Stuff I just made up</li>
<li>YMMV</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Don't distribute</h3><a id="user-content-dont-distribute" class="anchor" aria-label="Permalink: Don't distribute" href="#dont-distribute"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Rule 1: don't distribute where you don't have to
<ul dir="auto">
<li>Local systems have reliable primitives. Locks. Threads. Queues. Txns.
<ul dir="auto">
<li>When you move to a distributed system, you have to build from ground up.</li>
</ul>
</li>
<li>Is this thing small enough to fit on one node?
<ul dir="auto">
<li>"I have a big data problem"
<ul dir="auto">
<li>Softlayer will rent you a box with 3TB of ram for $5000/mo.</li>
<li>Supermicro will sell a 6TB box for ~$115,000 total.</li>
</ul>
</li>
</ul>
</li>
<li>Modern computers are FAST.
<ul dir="auto">
<li>Production JVM HTTP services I've known have pushed 50K requests/sec
<ul dir="auto">
<li>Parsing JSON events, journaling to disk, pushing to S3</li>
</ul>
</li>
<li>Protocol buffers over TCP: 10 million events/sec
<ul dir="auto">
<li>10-100 event batches/message, in-memory processing</li>
</ul>
</li>
</ul>
</li>
<li>Can this service tolerate a single node's guarantees?</li>
<li>Could we just stand up another one if it breaks?</li>
<li>Could manual intervention take the place of the distributed algorithm?</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Use an existing distributed system</h3><a id="user-content-use-an-existing-distributed-system" class="anchor" aria-label="Permalink: Use an existing distributed system" href="#use-an-existing-distributed-system"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>If we have to distribute, can we push the work onto some other software?
<ul dir="auto">
<li>What about a distributed database or log?</li>
<li>Can we pay Amazon to do this for us?</li>
<li>Conversely, what are the care and feeding costs?</li>
<li>How much do you have to learn to use/operate that distributed system?</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Never fail</h3><a id="user-content-never-fail" class="anchor" aria-label="Permalink: Never fail" href="#never-fail"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Buy really expensive hardware</li>
<li>Make changes to software and hardware in a controlled fashion
<ul dir="auto">
<li>Dry-run deployments against staging environments</li>
</ul>
</li>
<li>Possible to build very reliable networks and machines
<ul dir="auto">
<li>At the cost of moving slower, buying more expensive HW, finding talent</li>
<li>HW/network failure still <em>happens</em>, but sufficiently rare =&gt; low priority</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Accept failure</h3><a id="user-content-accept-failure" class="anchor" aria-label="Permalink: Accept failure" href="#accept-failure"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Distributed systems aren't just characterized by <em>latency</em>, but by
<em>recurrent, partial failure</em></li>
<li>Can we accept this failure and move on with our lives?
<ul dir="auto">
<li>What's our SLA anyway?</li>
<li>Can we recover by hand?</li>
<li>Can we pay someone to fix it?</li>
<li>Could insurance cover the damage?</li>
<li>Could we just call the customer and apologize?</li>
</ul>
</li>
<li>Sounds silly, but may be much cheaper
<ul dir="auto">
<li>We can never prevent 100% of system failures</li>
<li>Consciously choosing to recover <em>above</em> the level of the system</li>
<li>This is how financial companies and retailers do it!</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Recovery First</h3><a id="user-content-recovery-first" class="anchor" aria-label="Permalink: Recovery First" href="#recovery-first"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Assume a failure has just occurred: how will you recover?</li>
<li>Make this recovery the <em>default</em> path of execution</li>
<li>Writing recovery-first code keeps you from punting on error handling</li>
<li>Exercising recovery code by default means you know it works</li>
<li>Recovery by default means you don't have to worry about different semantics
during a real fault</li>
<li>If necessary, introduce a happy path for performance optimization
<ul dir="auto">
<li>But you lose some of these advantages!</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Reconciliation Loops</h3><a id="user-content-reconciliation-loops" class="anchor" aria-label="Permalink: Reconciliation Loops" href="#reconciliation-loops"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>You've got a complex, stateful system, and want to move it somewhere</li>
<li>Could devise a plan of changes, and apply those changes in order
<ul dir="auto">
<li>But what if some change breaks? How do you recover?</li>
</ul>
</li>
<li>Instead, maintain a <em>target</em>: a representation of what you <em>want</em> the system
to become</li>
<li>Next, write a function that looks at the current state, and <em>diffs</em> it with
the target</li>
<li>Use that diff to find a step that moves the system closer to the target</li>
<li>Repeat indefinitely</li>
<li>Robust to faults and interference
<ul dir="auto">
<li>What if your admin is tweaking things by hand?</li>
<li>What if two instances of the control system are running concurrently?</li>
</ul>
</li>
<li>Deployed to great effect in systems like <a href="https://queue.acm.org/detail.cfm?id=2898444" rel="nofollow">Borg &amp; Kubernetes</a></li>
<li>Also applicable to keeping data in sync between systems
<ul dir="auto">
<li>Making sure every order is shipped &amp; billed, for instance</li>
</ul>
</li>
<li>Again, we're looking for <em>monotonicity</em></li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Backups</h3><a id="user-content-backups" class="anchor" aria-label="Permalink: Backups" href="#backups"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Backups are essentially sequential consistency, BUT you lose a window of ops.
<ul dir="auto">
<li>When done correctly
<ul dir="auto">
<li>Some backup programs don't snapshot state, which leads to FS or DB
corruption</li>
<li>Broken fkey relationships, missing files, etc...</li>
</ul>
</li>
<li>Allow you to recover in a matter of minutes to days</li>
<li>But more than fault recovery, they allow you to step back in time
<ul dir="auto">
<li>Useful for recovering from logical faults
<ul dir="auto">
<li>Distributed DB did its job correctly, but you told it to delete key
data</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Redundancy</h3><a id="user-content-redundancy" class="anchor" aria-label="Permalink: Redundancy" href="#redundancy"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>OK, so failure is less of an option</li>
<li>Want to <em>reduce the probability of failure</em></li>
<li>Have the same state and same computation take place on several nodes
<ul dir="auto">
<li>I'm not a huge believer in active-spare
<ul dir="auto">
<li>Spare might have cold caches, broken disks, old versions, etc</li>
<li>Spares tend to fail when becoming active</li>
<li>Active-active wherever possible
<ul dir="auto">
<li>Predictability over efficiency</li>
</ul>
</li>
</ul>
</li>
<li>Also not a huge fan of only having 2 copies
<ul dir="auto">
<li>Node failure probabilities just too high</li>
<li>OK for not-important data</li>
<li>I generally want three copies of data
<ul dir="auto">
<li>For important stuff, 4 or 5</li>
<li>For Paxos and other majority-quorum systems, odd numbers: 3, 5, 7
common</li>
</ul>
</li>
</ul>
</li>
<li>Common DR strategy: Paxos across 5 nodes; 3 or 4 in primary DC
<ul dir="auto">
<li>Ops can complete as soon as the local nodes ack; low latencies</li>
<li>Resilient to single-node failure (though latencies will spike)</li>
<li>But you still have a sequentially consistent backup in the other DC
<ul dir="auto">
<li>So in the event you lose an entire DC, all's not lost</li>
</ul>
</li>
<li>See Camille Fournier's talks on ZK deployment</li>
</ul>
</li>
</ul>
</li>
<li>Redundancy improves availability so long as failures are uncorrelated
<ul dir="auto">
<li>Failures are not uncorrelated
<ul dir="auto">
<li>Disks from the same batch failing at the same time</li>
<li>Same-rack nodes failing when the top-of-rack switch blows</li>
<li>Same-DC nodes failing when the UPS blows</li>
<li>See entire EC2 AZ failures</li>
<li>Running the same bad computation on every node will break every node
<ul dir="auto">
<li>Expensive queries</li>
<li>Riak list-keys</li>
<li>Cassandra doomstones</li>
</ul>
</li>
<li>Cascading failures
<ul dir="auto">
<li>Thundering-herd</li>
<li>TCP incast</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Sharding</h3><a id="user-content-sharding" class="anchor" aria-label="Permalink: Sharding" href="#sharding"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>The problem is too big</li>
<li>Break the problem into parts small enough to fit on a node
<ul dir="auto">
<li>Not too small: small parts =&gt; high overhead</li>
<li>Not too big: need to rebalance work units gradually from node to node</li>
<li>Somewhere around 10-100 work units/node is ideal, IMO</li>
</ul>
</li>
<li>Ideal: work units of equal size
<ul dir="auto">
<li>Beware hotspots</li>
<li>Beware changing workloads with time</li>
</ul>
</li>
<li>Know your bounds in advance
<ul dir="auto">
<li>How big can a single part get before overwhelming a node?</li>
<li>How do we enforce that limit <em>before</em> it sinks a node in prod?
<ul dir="auto">
<li>Then sinks all the other nodes, one by one, as the system rebalances</li>
</ul>
</li>
</ul>
</li>
<li>Allocating shards to nodes
<ul dir="auto">
<li>Often built in to DB</li>
<li>Good candidate for ZK, Etcd, and so on</li>
<li>See Boundary's Ordasity</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Independent domains</h3><a id="user-content-independent-domains" class="anchor" aria-label="Permalink: Independent domains" href="#independent-domains"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Sharding is a specific case of a more general pattern: avoiding coordination
<ul dir="auto">
<li>Keep as much independent as possible
<ul dir="auto">
<li>Improves fault tolerance</li>
<li>Improves performance</li>
<li>Reduces complexity</li>
</ul>
</li>
<li>Sharding for scalability</li>
<li>Avoiding coordination via CRDTs</li>
<li>Flake IDs: <em>mostly</em> time-ordered identifiers, zero-coordination
<ul dir="auto">
<li>See <a href="http://yellerapp.com/posts/2015-02-09-flake-ids.html" rel="nofollow">http://yellerapp.com/posts/2015-02-09-flake-ids.html</a></li>
</ul>
</li>
<li>Partial availability: users can still use some parts of the system</li>
<li>Processing a queue: more consumers reduces the impact of expensive events</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">ID structure</h3><a id="user-content-id-structure" class="anchor" aria-label="Permalink: ID structure" href="#id-structure"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Things in our world have to have unique identifiers
<ul dir="auto">
<li>At scale, ID structure can make or break you</li>
<li>Consider your access patterns
<ul dir="auto">
<li>Scans</li>
<li>Sorts</li>
<li>Shards</li>
</ul>
</li>
<li>Sequential IDs require coordination: can you avoid them?
<ul dir="auto">
<li>Flake IDs, UUIDs, ...</li>
</ul>
</li>
<li>For <em>shardability</em>, can your ID map directly to a shard?</li>
<li>SaaS app: object ID can also encode customer ID</li>
<li>Twitter: tweet ID can encode user ID</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Immutable values</h3><a id="user-content-immutable-values" class="anchor" aria-label="Permalink: Immutable values" href="#immutable-values"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Data that never changes is trivial to store
<ul dir="auto">
<li>Never requires coordination</li>
<li>Cheap replication and recovery</li>
<li>Minimal repacking on disk</li>
</ul>
</li>
<li>Useful for Cassandra, Riak, any LSM-tree DB.
<ul dir="auto">
<li>Or for logs like Kafka!</li>
</ul>
</li>
<li>Easy to reason about: either present or it's not
<ul dir="auto">
<li>Eliminates all kinds of transactional headaches</li>
<li>Extremely cachable</li>
</ul>
</li>
<li>Extremely high availability and durability, tunable write latency
<ul dir="auto">
<li>Low read latencies: can respond from closest replica</li>
<li>Especially valuable for geographic distribution</li>
</ul>
</li>
<li>Requires garbage collection!
<ul dir="auto">
<li>But there are good ways to do this</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Mutable identities</h3><a id="user-content-mutable-identities" class="anchor" aria-label="Permalink: Mutable identities" href="#mutable-identities"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Pointers to immutable values</li>
<li>Pointers are small! Only metadata!
<ul dir="auto">
<li>Can fit huge numbers of pointers on a small DB</li>
<li>Good candidate for consensus services or relational DBs</li>
</ul>
</li>
<li>And typically, not many pointers in the system
<ul dir="auto">
<li>Your entire DB could be represented by a single pointer</li>
<li>Datomic only has ~5 identities</li>
</ul>
</li>
<li>Strongly consistent operations over identities can be <em>backed</em> by immutable
HA storage
<ul dir="auto">
<li>Take advantage of AP storage latencies and scale</li>
<li>Take advantage of strong consistency over small datasets provided by
consensus systems</li>
<li>Write availability limited by identity store
<ul dir="auto">
<li>But, reads eminently cachable if you only need sequential consistency</li>
<li>Can be even cheaper if you only need serializability</li>
</ul>
</li>
<li>See Rich Hickey's talks on Datomic architecture</li>
<li>See Pat Helland's 2013 RICON West keynote on Salesforce's storage</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Confluence</h3><a id="user-content-confluence" class="anchor" aria-label="Permalink: Confluence" href="#confluence"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Systems which are order-independent are easier to construct and reason about</li>
<li>Also helps us avoid coordination</li>
<li>CRDTs are confluent, which means we can apply updates without waiting</li>
<li>Immutable values are trivially confluent: once present, fixed</li>
<li>Streaming systems can leverage confluence as well:
<ul dir="auto">
<li>Buffer events, and compute+flush when you know you've seen everything</li>
<li>Emit partial results so you can take action now, e.g. for monitoring</li>
<li>When full data is available, merge with + or max</li>
<li>Bank ledgers are (mostly) confluent: txn order doesn't affect balance
<ul dir="auto">
<li>But when you need to enforce a minimum balance, no longer confluent</li>
<li>Combine with a sealing event (e.g. the day's end) to recover confluence</li>
</ul>
</li>
</ul>
</li>
<li>See Aiken, Widom, &amp; Hellerstein 1992, "Behavior of Database Production Rules"</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Backpressure</h3><a id="user-content-backpressure" class="anchor" aria-label="Permalink: Backpressure" href="#backpressure"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Services which talk to each other are usually connected by <em>queues</em></li>
<li>Service and queue capacity is finite</li>
<li>How do you handle it when a downstream service is unable to handle load?
<ol dir="auto">
<li>Consume resources and explode</li>
<li>Shed load. Start dropping requests.</li>
<li>Reject requests. Ignore the work and tell clients it failed.</li>
<li>Apply backpressure to clients, asking them to slow down.</li>
</ol>
</li>
<li>2-4 allow the system to catch up and recover
<ul dir="auto">
<li>But backpressure reduces the volume of work that has to be retried</li>
</ul>
</li>
<li>Backpressure defers choice to producers: compositional
<ul dir="auto">
<li>Clients of load-shedding systems are locked into load-shedding
<ul dir="auto">
<li>They have no way to tell that the system is hosed</li>
</ul>
</li>
<li>Clients of backpressure systems can apply backpressure to <em>their clients</em>
<ul dir="auto">
<li>Or shed load, if they choose</li>
</ul>
</li>
<li>If you're making an asynchronous system, <em>always</em> include backpressure
<ul dir="auto">
<li>Your users will thank you later</li>
</ul>
</li>
</ul>
</li>
<li>Fundamentally: <em>bounding resources</em>
<ul dir="auto">
<li>Request timeouts (bounded time)</li>
<li>Exponential backoffs (bounded use)</li>
<li>Bounded queues</li>
<li>Bounded concurrency</li>
</ul>
</li>
<li>See Zach Tellman, "Everything Will Flow"</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Services for domain models</h3><a id="user-content-services-for-domain-models" class="anchor" aria-label="Permalink: Services for domain models" href="#services-for-domain-models"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>The problem is composed of interacting logical pieces</li>
<li>Pieces have distinct code, performance, storage needs
<ul dir="auto">
<li>Monolithic applications are essentially <em>multitenant</em> systems
<ul dir="auto">
<li>Multitenancy is tough</li>
<li>But its often okay to run multiple logical "services" in the same process</li>
</ul>
</li>
</ul>
</li>
<li>Divide your system into logical services for discrete parts of the domain
model
<ul dir="auto">
<li>OO approach: each <em>noun</em> is a service
<ul dir="auto">
<li>User service</li>
<li>Video service</li>
<li>Index service</li>
</ul>
</li>
<li>Functional approach: each <em>verb</em> is a service
<ul dir="auto">
<li>Auth service</li>
<li>Search service</li>
<li>Dispatch/routing service</li>
</ul>
</li>
<li>Most big systems I know of use a hybrid
<ul dir="auto">
<li>Services for nouns is a good way to enforce <em>datatype invariants</em></li>
<li>Services for verbs is a good way to enforce <em>transformation invariants</em></li>
<li>So have a basic User service, which is used <em>by</em> an Auth service</li>
</ul>
</li>
<li>Where you draw the line... well that's tricky
<ul dir="auto">
<li>Services come with overhead: have as few as possible</li>
<li>Consider work units</li>
<li>Separate services which need to scale independently</li>
<li>Colocate services with tight dependencies and tight latency budgets</li>
<li>Colocate services which use complementary resources (e.g. disk and CPU)
<ul dir="auto">
<li>By hand: Run memcache on rendering nodes</li>
<li>Newer shops: Google Borg, Mesos, Kubernetes</li>
</ul>
</li>
</ul>
</li>
<li>Services should encapsulate and abstract
<ul dir="auto">
<li>Try to build trees instead of webs</li>
<li>Avoid having outsiders manipulate a service's data store directly</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Structure Follows Social Spaces</h3><a id="user-content-structure-follows-social-spaces" class="anchor" aria-label="Permalink: Structure Follows Social Spaces" href="#structure-follows-social-spaces"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Production software is a fundamentally social artifact</li>
<li>Natural alignment: a team or person owns a specific service
<ul dir="auto">
<li>Jo Freeman, "The Tyranny of Structurelessness"
<ul dir="auto">
<li><a href="https://www.jofreeman.com/joreen/tyranny.htm" rel="nofollow">https://www.jofreeman.com/joreen/tyranny.htm</a></li>
<li>Responsibility and power should be explicit</li>
<li>Rotate people through roles to prevent fiefdoms
<ul dir="auto">
<li>People own <em>problem spaces</em>, not services</li>
<li>Promotes information sharing</li>
</ul>
</li>
<li>But don't rotate too often
<ul dir="auto">
<li>Ramp-up costs in software are very high</li>
</ul>
</li>
</ul>
</li>
<li>Pair Freeman with Downs' "Inside Bureaucracy"
<ul dir="auto">
<li><a href="https://www.rand.org/content/dam/rand/pubs/papers/2008/P2963.pdf" rel="nofollow">https://www.rand.org/content/dam/rand/pubs/papers/2008/P2963.pdf</a></li>
</ul>
</li>
</ul>
</li>
<li>As the team grows, its mission and thinking will formalize
<ul dir="auto">
<li>So too will services and their boundaries</li>
<li>Gradually accruing body of assumptions about service relation to the world</li>
<li>Punctuated by rewrites to respond to changing external pressures</li>
<li>Tushman &amp; Romanelli, 1985: Organizational Transformation as Punctuated Equilibrium</li>
</ul>
</li>
<li>Services can be libraries
<ul dir="auto">
<li>Initially, <em>all</em> your services should be libraries</li>
<li>Perfectly OK to depend on a user library in multiple services</li>
<li>Libraries with well-defined boundaries are easy to extract into
services later</li>
</ul>
</li>
<li>Social structure governs the library/service boundary
<ul dir="auto">
<li>With few users of a library, or tightly-coordinated users, changes are easy</li>
<li>But across many teams, users have varying priorities and must be convinced</li>
<li>Why should users do work to upgrade to a new library version?</li>
<li>Services <em>force</em> coordination through a defined API deprecation lifecycle
<ul dir="auto">
<li>You can also enforce this with libraries through code review &amp; tooling</li>
</ul>
</li>
</ul>
</li>
<li>Services enable centralized control
<ul dir="auto">
<li>Your performance improvements affect everyone instantly</li>
<li>Gradually shift to a new on-disk format or backing database</li>
<li>Instrument use of the service in one place</li>
<li>Harder to do these things with libraries</li>
</ul>
</li>
<li>Services have costs
<ul dir="auto">
<li>The failure-complexity and latency overhead of a network call</li>
<li>Tangled food web of service dependencies</li>
<li>Hard to statically analyze codepaths</li>
<li>You thought library API versioning was hard</li>
<li>Additional instrumentation/deployment</li>
</ul>
</li>
<li>Services can use good client libraries
<ul dir="auto">
<li>That library might be "Open a socket" or an HTTP client
<ul dir="auto">
<li>Leverage HTTP headers!
<ul dir="auto">
<li>Accept headers for versioning</li>
<li>Lots of support for caching and proxying</li>
</ul>
</li>
<li>Haproxy is an excellent router for both HTTP and TCP services</li>
</ul>
</li>
<li>Eventually, library might include mock IO
<ul dir="auto">
<li>Service team is responsible for testing that the service provides an API</li>
<li>When the API is known to be stable, every client can <em>assume</em> it works</li>
<li>Removes the need for network calls in test suites</li>
<li>Dramatic reduction in test runtime and dev environment complexity</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Cross-service coordination</h3><a id="user-content-cross-service-coordination" class="anchor" aria-label="Permalink: Cross-service coordination" href="#cross-service-coordination"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Coordination between services requires special protocols
<ul dir="auto">
<li>Have to re-invent transactions</li>
<li>Go commutative where possible</li>
<li>Sagas
<ul dir="auto">
<li>Was written for a single-node world: we have to be clever in distributed contexts</li>
<li>Transactions must be idempotent, OR commute with rollbacks</li>
</ul>
</li>
<li><a href="http://www.cs.ucsb.edu/~vaibhavarora/Typhon-Ieee-Cloud-2017.pdf" rel="nofollow">Typhon/Cerberus</a>
<ul dir="auto">
<li>Protocols for causal consistency over multiple data stores
<ul dir="auto">
<li>e.g. If Lupita blocks Miss Angela, then posts, Miss Angela can't see it</li>
</ul>
</li>
<li>Typhon: A single logical <em>entity</em> has <em>data item</em> representations in
different data stores
<ul dir="auto">
<li>Assumes datastores are serializable or offer atomic read/cas on items</li>
<li>Transactions which access same entity AND T1 happens-before T2 get a
causal dependency edge T1 -&gt; T2</li>
</ul>
</li>
<li>Cerberus: protocol for transactions involving a single entity x
<ul dir="auto">
<li>Writes can only affect one representation of x</li>
<li>Any number of reads across representations of x</li>
<li>Global metadata: a version vector for each entity (GVV)</li>
<li>Each representation metadata:
<ul dir="auto">
<li>Update version vector (UVV): versions known as of last update</li>
<li>Read version vector (RVV): versions known as of last read</li>
</ul>
</li>
<li>Conflicts detected when GVV &lt; UVV/RVV</li>
<li>Two phases:
<ul dir="auto">
<li>Reads
<ul dir="auto">
<li>Check GVV for entity x</li>
<li>Perform reads of x on each (asked-for) representation</li>
<li>At each representation: check RVV &lt;= GVV, update RVV</li>
</ul>
</li>
<li>Writes
<ul dir="auto">
<li>Send write to representation</li>
<li>Check UVV &lt;= GVV &amp; RVV &lt;= GVV</li>
</ul>
</li>
<li>Commit
<ul dir="auto">
<li>Update representation and RVV/UVV ensuring RVV/UVV unchanged</li>
<li>New UVV constructed by incrementing i'th entry of existing UVV</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li>General-purpose transactions
<ul dir="auto">
<li><a href="http://cs.yale.edu/homes/thomson/publications/calvin-sigmod12.pdf" rel="nofollow">Calvin</a>
<ul dir="auto">
<li>Serializable (or strict-1SR) transactions</li>
<li>Deterministic txns enqueued into sharded global log</li>
<li>Log ensures txn order</li>
<li>Application at replicas/shards requires no further coordination</li>
<li>Minimum latency floor for log windows</li>
</ul>
</li>
<li><a href="https://www.cockroachlabs.com/guides/cockroachdb-the-resilient-geo-distributed-sql-database-sigmod-2020/" rel="nofollow">CockroachDB</a>
<ul dir="auto">
<li>Serializable</li>
<li>Assumes linearizable stores</li>
<li>Assumes semi-sync clocks</li>
<li>Similar to a more tractable Spanner</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Migrations</h3><a id="user-content-migrations" class="anchor" aria-label="Permalink: Migrations" href="#migrations"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Migrations are hard.
<ul dir="auto">
<li>There's no silver bullet</li>
<li>But some techniques can make your life easier</li>
</ul>
</li>
<li>Hard cut
<ul dir="auto">
<li>Write new system and migration to copy old data to it</li>
<li>Have dependent services talk to both--but in practice, only one.</li>
<li>Turn off old system</li>
<li>Copy data</li>
<li>Start up new system</li>
<li>Tradeoffs!
<ul dir="auto">
<li>Don't have to worry about in-flight data</li>
<li>Simple migration scripts: just read all data and write to new datastore</li>
<li>Requires downtime proportional to migration script</li>
</ul>
</li>
<li>Can sometimes scope this to a single shard/user/domain at a time</li>
</ul>
</li>
<li>Incremental
<ul dir="auto">
<li>Write new system B</li>
<li>Deploy alongside original A</li>
<li>Dependent services talk to both
<ul dir="auto">
<li>Ideal: find all readers, have every reader talk to both A and B</li>
<li>Then start writing to B
<ul dir="auto">
<li>This frees you from having to worry about readers who only know about A</li>
</ul>
</li>
</ul>
</li>
<li>Consistency nightmares; need to trace all data dependencies</li>
<li>Tradeoffs!
<ul dir="auto">
<li>Reduced/no downtime</li>
<li>But complex reasoning about data dependencies required</li>
</ul>
</li>
</ul>
</li>
<li>Wrapper services
<ul dir="auto">
<li>Operationally speaking, it can be tricky to FIND and change all users of A</li>
<li>So... don't. Introduce a wrapper service W which proxies to A</li>
<li>Introduce B, and make changes so that W talks to B as well</li>
<li>When A is phased out, remove W and talk directly to B.</li>
<li>Allows for centralized metrics, errors, comparison of behavior, etc</li>
</ul>
</li>
<li>Eventual atomicity
<ul dir="auto">
<li>Imagine you write each update to old service A, then new service B</li>
<li>At some point, your write to A will succeed, and B will fail. What then?</li>
<li>Can use read-repair: read A and B, fill in missing updates
<ul dir="auto">
<li>But this requires mergeability: only works for things like CRDTs</li>
</ul>
</li>
<li>Can use a reconciliation process
<ul dir="auto">
<li>Iterate over whole DB, look for changes, apply to both.</li>
<li>Also requires something like a CRDT</li>
</ul>
</li>
<li>Can use a Saga
<ul dir="auto">
<li>All updates go to durable queue</li>
<li>Queue worker retries until updates applied to A and B</li>
<li>Might require ordering updates to avoid state divergence
<ul dir="auto">
<li>Potentially <em>global</em> serialization</li>
</ul>
</li>
<li>Be aware of the DB's consistency model</li>
</ul>
</li>
</ul>
</li>
<li>Isolation
<ul dir="auto">
<li>Imagine Jane writes w1 to A, then B</li>
<li>Concurrently, Naomi writes w2 to B, then A</li>
<li>Result: A = w2, B = w1</li>
<li>Eventual atomicity isn't enough to prevent divergence</li>
<li>Can reduce issues by picking a standard order: always A then B (or B then A)
<ul dir="auto">
<li>But imagine
<ul dir="auto">
<li>Jane writes A = w1</li>
<li>Naomi writes A = w2</li>
<li>Naomi writes B = w2</li>
<li>Jane writes B = w1</li>
</ul>
</li>
<li>We've got a mixed result again. Shoot.</li>
</ul>
</li>
<li>Can mitigate using CRDTs</li>
<li>Or, if A and B are sequentially consistent, can use a CaS operation to
ensure agreement on order
<ul dir="auto">
<li>"Write w2 iff the last write was w1"</li>
</ul>
</li>
<li>If operations affect multiple keys, the CaS logic has to be applied at that
level too</li>
</ul>
</li>
<li>Helpful properties for incremental migrations
<ul dir="auto">
<li>Determinism
<ul dir="auto">
<li>Avoid having the DB generate random numbers, automatic IDs, timestamps</li>
<li>Easier to apply updates to two datastores and get the same results</li>
</ul>
</li>
<li>Idempotence
<ul dir="auto">
<li>Lets you retry updates freely</li>
</ul>
</li>
<li>Commutativity
<ul dir="auto">
<li>Removes the need for serializing updates</li>
</ul>
</li>
<li>CRDTs: associativity, commutativity, idempotence.</li>
<li>Immutability: trivial CRDTs</li>
<li>Statelessness: no state to worry about!
<ul dir="auto">
<li>Just make sure you talk to external stateful stuff the same way</li>
</ul>
</li>
</ul>
</li>
<li>What about swapping out queues?
<ul dir="auto">
<li>As we've touched, queue systems should already be designed for idempotency,
and ideally commutativity
<ul dir="auto">
<li>If so, this is (relatively) easy</li>
<li>Workers consume from both queues</li>
<li>Flip producers to send messages only to the new queue</li>
<li>Wait for old queue to be exhausted</li>
<li>Decommission old queue</li>
</ul>
</li>
<li>But we WANTED order???
<ul dir="auto">
<li>You're going to need to reconstruct it</li>
<li>One option: single producer tightly coupled to queue
<ul dir="auto">
<li>Writes each message m to both A and B; doesn't move on until both ack</li>
<li>This enforces that both A and B agree on order</li>
<li>Consumers can treat A and B identically: consume just from A or B
<ul dir="auto">
<li>Again, assumes idempotence!</li>
</ul>
</li>
</ul>
</li>
<li>Another option: sequence numbers, order reconstructed at client
<ul dir="auto">
<li>e.g. assign sequence numbers to each value
<ul dir="auto">
<li>Use the queue offsets from A?</li>
<li>Use a consensus system?</li>
</ul>
</li>
<li>Clients read sequence numbers, store in internal buffer, apply in order</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Review</h3><a id="user-content-review-5" class="anchor" aria-label="Permalink: Review" href="#review-5"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">When possible, try to use a single node instead of a distributed system. Accept
that some failures are unavoidable: SLAs and apologies can be cost-effective.
To handle catastrophic failure, we use backups. To improve reliability, we
introduce redundancy. To scale to large problems, we divide the problem into
shards. Immutable values are easy to store and cache, and can be referenced by
mutable identities, allowing us to build strongly consistent systems at large
scale. As software grows, different components must scale independently,
and we break out libraries into distinct services. Service structure goes
hand-in-hand with teams.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Production Concerns</h2><a id="user-content-production-concerns" class="anchor" aria-label="Permalink: Production Concerns" href="#production-concerns"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>More than design considerations</li>
<li>Proofs are important, but real systems do IO</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Distributed systems are supported by your culture</h3><a id="user-content-distributed-systems-are-supported-by-your-culture" class="anchor" aria-label="Permalink: Distributed systems are supported by your culture" href="#distributed-systems-are-supported-by-your-culture"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Understanding a distributed system in production requires close cooperation
of people with many roles
<ul dir="auto">
<li>Development</li>
<li>QA</li>
<li>Operations</li>
</ul>
</li>
<li>Empathy matters
<ul dir="auto">
<li>Developers have to care about production</li>
<li>Ops has to care about implementation</li>
<li>Good communication enables faster diagnosis</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Test everything</h3><a id="user-content-test-everything" class="anchor" aria-label="Permalink: Test everything" href="#test-everything"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Type systems are great for preventing logical errors
<ul dir="auto">
<li>Which reduces your testing burden</li>
</ul>
</li>
<li>However, they are <em>not</em> great at predicting or controlling runtime
performance</li>
<li>So, you need a solid test suite
<ul dir="auto">
<li>Ideally, you want a <em>slider</em> for rigorousness</li>
<li>Quick example-based tests that run in a few seconds</li>
<li>More thorough property-based tests that can run overnight</li>
<li>Be able to simulate an entire cluster in-process</li>
<li>Control concurrent interleavings with simulated networks</li>
<li>Automated hardware faults</li>
</ul>
</li>
<li>Testing distributed systems is much, much harder than testing local ones
<ul dir="auto">
<li>Huge swath of failure modes you've never even heard of</li>
<li>Combinatorial state spaces</li>
<li>Bugs can manifest only for small/large/intermediate time/space/concurrency</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">"It's Slow"</h3><a id="user-content-its-slow" class="anchor" aria-label="Permalink: &quot;It's Slow&quot;" href="#its-slow"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Jeff Hodges: The worst bug you'll ever hear is "it's slow"
<ul dir="auto">
<li>Happens all the time, really difficult to localize</li>
<li>Because the system is distributed, have to profile multiple nodes
<ul dir="auto">
<li>Not many profilers are built for this</li>
<li>Sigelman et al, 2010: Dapper, a Large-Scale Distributed Systems Tracing
Infrastructure</li>
<li>Zipkin</li>
<li>Big tooling investment</li>
</ul>
</li>
<li>Profilers are good at finding CPU problems
<ul dir="auto">
<li>But high latency is often a sign of IO, not CPU</li>
<li>Disk latency</li>
<li>Network latency</li>
<li>GC latency</li>
<li>Queue latency</li>
</ul>
</li>
<li>Try to localize problem using application-level metrics
<ul dir="auto">
<li>Then dig in to process and OS performance</li>
</ul>
</li>
<li>Latency variance between nodes doing the same work is an important signal
<ul dir="auto">
<li>1/3 nodes slow: likely node HW, re-route</li>
<li>3/3 nodes slow: likely a logical fault: look at shard size, workload, queries</li>
</ul>
</li>
<li>Tail latencies are magnified by fanout workloads
<ul dir="auto">
<li><a href="https://research.google.com/pubs/pub40801.html" rel="nofollow">Jeff Dean, 2013: The Tail at Scale</a></li>
<li>Consider speculative parallelism</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Instrument everything</h3><a id="user-content-instrument-everything" class="anchor" aria-label="Permalink: Instrument everything" href="#instrument-everything"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Slowness (and outright errors) in prod stem from the interactions <em>between</em>
systems
<ul dir="auto">
<li>Why? Because your thorough test suite probably verified that the single
system was mostly correct</li>
<li>So we need a way to understand what the system is doing in prod
<ul dir="auto">
<li>In relation to its dependencies</li>
<li>Which can, in turn, drive new tests</li>
</ul>
</li>
<li>In a way, good monitoring is like continuous testing</li>
<li>But not a replacement: these are distinct domains</li>
<li>Both provide assurance that your changes are OK</li>
<li>Want high-frequency monitoring
<ul dir="auto">
<li>Production behaviors can take place on 1ms scales
<ul dir="auto">
<li>TCP incast</li>
<li>~1ms resolution <em>ideally</em></li>
</ul>
</li>
<li>Ops response time, in the limit, scales linearly with observation latency
<ul dir="auto">
<li>~1 second end to end latency</li>
</ul>
</li>
<li>Ideally, millisecond latencies, maybe ms resolution too
<ul dir="auto">
<li>Usually cost-prohibitive; back off to 1s or 10s</li>
<li>Sometimes you can tolerate 60s</li>
</ul>
</li>
</ul>
</li>
<li>And for capacity planning, hourly/daily seasonality is more useful</li>
<li>Instrumentation should be tightly coupled to the app
<ul dir="auto">
<li>Measure only what matters
<ul dir="auto">
<li>Responding to requests is important</li>
<li>Node CPU doesn't matter as much</li>
</ul>
</li>
<li>Key metrics for most systems
<ul dir="auto">
<li>Apdex: successful response WITHIN latency SLA</li>
<li>Latency profiles: 0, 0.5, 0.95, 0.99, 1
<ul dir="auto">
<li>Percentiles, not means</li>
<li>BTW you can't take the mean of percentiles either</li>
</ul>
</li>
<li>Overall throughput</li>
<li>Queue statistics</li>
<li>Queries per query</li>
<li>Subjective experience of other systems latency/throughput
<ul dir="auto">
<li>The DB might think it's healthy, but clients could see it as slow</li>
<li>Combinatorial explosion--best to use this when drilling into a failure</li>
</ul>
</li>
</ul>
</li>
<li>You probably have to write this instrumentation yourself
<ul dir="auto">
<li>Invest in a metrics library</li>
</ul>
</li>
</ul>
</li>
<li>Out-of-the-box monitoring usually doesn't measure what really matters: your
app's behavior
<ul dir="auto">
<li>But it can be really useful in tracking down causes of problems</li>
<li>Host metrics like CPU, disk, etc</li>
<li>Where your app does something common (e.g. rails apps) tools like New
Relic work well</li>
</ul>
</li>
<li>Shard metrics by client
<ul dir="auto">
<li>Helpful when users have varying workloads</li>
<li>Can tune thresholds to be appropriate for that client</li>
<li>A few for major clients, another bucket for "the rest"</li>
</ul>
</li>
<li>Superpower: distributed tracing infra (Zipkin, Dapper, etc)
<ul dir="auto">
<li>Significant time investment</li>
<li><a href="https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-chow.pdf" rel="nofollow">Mystery Machine</a>
<ul dir="auto">
<li>Automatic inference of causal relationships between services from trace data</li>
<li>Identification of critical paths</li>
<li>Performance modeling new algorithms before implementation</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Logging</h3><a id="user-content-logging" class="anchor" aria-label="Permalink: Logging" href="#logging"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Logging is less useful at scale
<ul dir="auto">
<li>Problems may not be localized to one node
<ul dir="auto">
<li>As requests touch more services, must trace through many logfiles</li>
<li>Invest in log collection infrastructure
<ul dir="auto">
<li>ELK, Splunk, etc</li>
</ul>
</li>
</ul>
</li>
<li>Unstructured information is harder to aggregate
<ul dir="auto">
<li>Log structured events</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Shadow traffic</h3><a id="user-content-shadow-traffic" class="anchor" aria-label="Permalink: Shadow traffic" href="#shadow-traffic"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Load tests are only useful insofar as the simulated load matches the actual
load</li>
<li>Consider dumping production traffic
<ul dir="auto">
<li>Awesome: kill a process with SIGUSR1, it dumps five minutes of request load</li>
<li>Awesome: tcpdump/tcpreplay harnesses for requests</li>
<li>Awesome: shadowing live prod traffic to your staging/QA nodes</li>
</ul>
</li>
<li>See Envoy from Lyft</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Versioning</h3><a id="user-content-versioning" class="anchor" aria-label="Permalink: Versioning" href="#versioning"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Protocol versioning is, as far as I know, a wide-open problem
<ul dir="auto">
<li>Do include a version tag with all messages</li>
<li>Do include compatibility logic</li>
<li>Inform clients when their request can't be honored
<ul dir="auto">
<li>And instrument this so you know which systems have to be upgraded</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Rollouts</h3><a id="user-content-rollouts" class="anchor" aria-label="Permalink: Rollouts" href="#rollouts"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Rollouts are often how you fix problems</li>
<li>Spend the time to get automated, reliable deploys
<ul dir="auto">
<li>Amplifies everything else you do</li>
<li>Have nodes smoothly cycle through to prevent traffic interruption
<ul dir="auto">
<li>This implies you'll have multiple versions of your software running at
once
<ul dir="auto">
<li>Versioning rears its ugly head</li>
</ul>
</li>
</ul>
</li>
<li>Inform load balancer that they're going out of rotation</li>
<li>Coordinate to prevent cascading failures</li>
</ul>
</li>
<li>Roll out only to a fraction of load or fraction of users
<ul dir="auto">
<li>Gradually ramp up number of users on the new software</li>
<li>Either revert or roll forward when you see errors</li>
<li>Consider shadowing traffic in prod and comparing old/new versions
<ul dir="auto">
<li>Good way to determine if new code is faster &amp; correct</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Automated Control</h3><a id="user-content-automated-control" class="anchor" aria-label="Permalink: Automated Control" href="#automated-control"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Automated failure handling is good</li>
<li>But not too much
<ul dir="auto">
<li>"Ironies of Automation", Bainbridge 1983
<ul dir="auto">
<li><a href="https://pdfs.semanticscholar.org/0713/bb9d9b138e4e0a15406006de9b0cddf68e28.pdf" rel="nofollow">https://pdfs.semanticscholar.org/0713/bb9d9b138e4e0a15406006de9b0cddf68e28.pdf</a></li>
</ul>
</li>
<li>Controlling complex systems requires operators with accurate mental models
of how the system is behaving, ought to behave, and how it responds to
control inputs
<ul dir="auto">
<li>If you aren't engaged in the control process, you forget these things</li>
<li>Human takeover is then challenging!</li>
</ul>
</li>
<li>Humans literally cannot pay attention to things that don't change for more
than ~30 minutes</li>
<li>Merely monitoring, as opposed to running, a complex process "deskills"
operators</li>
<li>So... consider <em>deliberately</em> bringing humans into the control process</li>
<li>Catastrophic failure is easy to identify
<ul dir="auto">
<li>But automated control tends to mask failure: prefailure trends are not
apparent until well outside the automated-control regime</li>
</ul>
</li>
<li>If humans are expected to verify automated control decisions, humans must
be <em>capable</em> of verifying that reasoning
<ul dir="auto">
<li>Huge ML models (for example) make inscrutable decisions</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Feature flags</h3><a id="user-content-feature-flags" class="anchor" aria-label="Permalink: Feature flags" href="#feature-flags"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>We want incremental rollouts of a changeset after a deploy
<ul dir="auto">
<li>Introduce features one by one to watch their impact on metrics</li>
<li>Gradually shift load from one database to another</li>
<li>Disable features when rollout goes wrong</li>
</ul>
</li>
<li>We want to obtain partial availability when some services are degraded
<ul dir="auto">
<li>Disable expensive features to speed recovery during a failure</li>
</ul>
</li>
<li>Use a highly available coordination service to decide which codepaths to
enable, or how often to take them
<ul dir="auto">
<li>This service should have minimal dependencies
<ul dir="auto">
<li>Don't use the primary DB</li>
</ul>
</li>
</ul>
</li>
<li>When things go wrong, you can <em>tune</em> the system's behavior
<ul dir="auto">
<li>When coordination service is down, fail <em>safe</em>!</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Chaos engineering</h3><a id="user-content-chaos-engineering" class="anchor" aria-label="Permalink: Chaos engineering" href="#chaos-engineering"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Breaking things in production
<ul dir="auto">
<li>Forces engineers to handle failure appropriately <em>now</em>, not in response to
an incident later</li>
<li>Identifies unexpected dependencies in the critical path
<ul dir="auto">
<li>"When the new stats service goes down, it takes the API with it. Are you
<em>sure</em> that's necessary?"</li>
</ul>
</li>
<li>Requires good instrumentation and alerting, so you can measure impact of
events</li>
<li>Limited blast radius
<ul dir="auto">
<li>Don't nuke an entire datacenter every five minutes
<ul dir="auto">
<li>But <em>do</em> try it once a quarter</li>
</ul>
</li>
<li>Don't break <em>too</em> many nodes in a replication group</li>
<li>Break only a small fraction of requests/users at a time</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Oh no, queues</h3><a id="user-content-oh-no-queues" class="anchor" aria-label="Permalink: Oh no, queues" href="#oh-no-queues"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Every queue is a place for things to go horribly, horribly wrong
<ul dir="auto">
<li>No node has unbounded memory. Your queues <em>must</em> be bounded</li>
<li>But how big? Nobody knows</li>
<li>Instrument your queues in prod to find out</li>
</ul>
</li>
<li>Little's Law: mean queue depth = mean arrival rate * mean latency
<ul dir="auto">
<li>This is distribution-independent!</li>
</ul>
</li>
<li>Queues exist to smooth out fluctuations in load
<ul dir="auto">
<li>Improves throughput at expense of latency</li>
<li>If your load is higher than capacity, no queue will save you
<ul dir="auto">
<li><a href="https://ferd.ca/queues-don-t-fix-overload.html" rel="nofollow">https://ferd.ca/queues-don-t-fix-overload.html</a></li>
<li>Shed load or apply backpressure when queues become full</li>
<li>Instrument this
<ul dir="auto">
<li>When load-shedding occurs, alarm bells should ring</li>
<li>Backpressure is visible as upstream latency</li>
</ul>
</li>
</ul>
</li>
<li>Instrument queue depths
<ul dir="auto">
<li>High depths is a clue that you need to add node capacity
<ul dir="auto">
<li>End to end queue latency should be smaller than fluctuation timescales</li>
</ul>
</li>
<li>Raising the queue size can be tempting, but is a vicious cycle</li>
</ul>
</li>
<li>All of this is HARD. I don't have good answers for you
<ul dir="auto">
<li>Ask Jeff Hodges why it's hard: see his RICON West 2013 talk</li>
<li>See Zach Tellman - Everything Will Flow</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Review</h2><a id="user-content-review-6" class="anchor" aria-label="Permalink: Review" href="#review-6"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Running distributed systems requires cooperation between developers, QA, and
operations engineers. Static analysis, and a test suite including example- and
property-based tests, can help ensure program correctness, but understanding
production behavior requires comprehensive instrumentation and alerting. Mature
distributed systems teams often invest in tooling: traffic shadowing,
versioning, incremental deploys, and feature flags. Finally, queues require
special care.</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Further reading</h2><a id="user-content-further-reading" class="anchor" aria-label="Permalink: Further reading" href="#further-reading"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Online</h3><a id="user-content-online" class="anchor" aria-label="Permalink: Online" href="#online"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Mixu has a delightful book on distributed systems with incredible detail. <a href="http://book.mixu.net/distsys/" rel="nofollow">http://book.mixu.net/distsys/</a></li>
<li>Jeff Hodges has some excellent, production-focused advice. <a href="https://www.somethingsimilar.com/2013/01/14/notes-on-distributed-systems-for-young-bloods/" rel="nofollow">https://www.somethingsimilar.com/2013/01/14/notes-on-distributed-systems-for-young-bloods/</a> <a href="https://player.vimeo.com/video/42898664" rel="nofollow">https://player.vimeo.com/video/42898664</a></li>
<li>The Fallacies of Distributed Computing is a classic text on mistaken assumptions we make designing distributed systems. <a href="http://www.rgoarchitects.com/Files/fallacies.pdf" rel="nofollow">http://www.rgoarchitects.com/Files/fallacies.pdf</a></li>
<li>Christopher Meiklejohn has a list of key papers in distributed systems. <a href="http://christophermeiklejohn.com/distributed/systems/2013/07/12/readings-in-distributed-systems.html" rel="nofollow">http://christophermeiklejohn.com/distributed/systems/2013/07/12/readings-in-distributed-systems.html</a></li>
<li>Dan Creswell has a lovely reading list. <a href="https://dancres.github.io/Pages/" rel="nofollow">https://dancres.github.io/Pages/</a></li>
</ul>
<div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Trees</h3><a id="user-content-trees" class="anchor" aria-label="Permalink: Trees" href="#trees"><svg data-component="Octicon" class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<ul dir="auto">
<li>Martin Kleppmann's <a href="https://dataintensive.net/" rel="nofollow">Designing Data-Intensive
Applications</a> offers a thorough tour of
distributed systems for practitioners.</li>
<li>Nancy Lynch's "Distributed Algorithms" is a comprehensive overview of the
field from a more theoretical perspective</li>
<li>Suggestions from students:
<ul dir="auto">
<li>Donella Meadows' "Thinking in Systems"</li>
<li>"Database Internals: A Deep Dive into How Distributed Data Systems Work"
<ul dir="auto">
<li>Perhaps more intense than Kleppman</li>
</ul>
</li>
<li>"Thinking in Systems"</li>
</ul>
</li>
</ul>
</article></div><div class="d-none"></div></section></div></div></div> </div> <!-- --> </div></div></div></div></div></div><div class="ScrollMarksContainer-module__scrollMarksContainer__Eu7uU" id="find-result-marks-container"></div><div class="d-none"></div><div class="d-none"></div></div> <!-- --> <!-- --> </div>
</react-app>
</div>
</turbo-frame>
</main>
</div>
</div>
<footer class="footer f6 color-fg-muted color-border-subtle tmp-pt-7 tmp-pb-6 p-responsive" role="contentinfo" >
<h2 class='sr-only'>Footer</h2>
<div class="d-flex flex-justify-center flex-items-center flex-column-reverse flex-lg-row flex-wrap flex-lg-nowrap">
<div class="d-flex flex-items-center flex-shrink-0 mx-2">
<a aria-label="GitHub Homepage" class="footer-octicon mr-2" href="https://github.com">
<svg aria-hidden="true" data-component="Octicon" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-mark-github">
<path d="M10.226 17.284c-2.965-.36-5.054-2.493-5.054-5.256 0-1.123.404-2.336 1.078-3.144-.292-.741-.247-2.314.09-2.965.898-.112 2.111.36 2.83 1.01.853-.269 1.752-.404 2.853-.404 1.1 0 1.999.135 2.807.382.696-.629 1.932-1.1 2.83-.988.315.606.36 2.179.067 2.942.72.854 1.101 2 1.101 3.167 0 2.763-2.089 4.852-5.098 5.234.763.494 1.28 1.572 1.28 2.807v2.336c0 .674.561 1.056 1.235.786 4.066-1.55 7.255-5.615 7.255-10.646C23.5 6.188 18.334 1 11.978 1 5.62 1 .5 6.188.5 12.545c0 4.986 3.167 9.12 7.435 10.669.606.225 1.19-.18 1.19-.786V20.63a2.9 2.9 0 0 1-1.078.224c-1.483 0-2.359-.808-2.987-2.313-.247-.607-.517-.966-1.034-1.033-.27-.023-.359-.135-.359-.27 0-.27.45-.471.898-.471.652 0 1.213.404 1.797 1.235.45.651.921.943 1.483.943.561 0 .92-.202 1.437-.719.382-.381.674-.718.944-.943"></path>
</svg>
</a>
<span>
&copy; 2026 GitHub,&nbsp;Inc.
</span>
</div>
<nav aria-label="Footer">
<h3 class="sr-only" id="sr-footer-heading">Footer navigation</h3>
<ul class="list-style-none d-flex flex-justify-center flex-wrap mb-2 mb-lg-0" aria-labelledby="sr-footer-heading">
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to Terms&quot;,&quot;label&quot;:&quot;text:terms&quot;}" href="https://docs.github.com/site-policy/github-terms/github-terms-of-service" data-view-component="true" class="Link--secondary Link">Terms</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to privacy&quot;,&quot;label&quot;:&quot;text:privacy&quot;}" href="https://docs.github.com/site-policy/privacy-policies/github-privacy-statement" data-view-component="true" class="Link--secondary Link">Privacy</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to security&quot;,&quot;label&quot;:&quot;text:security&quot;}" href="https://github.com/security" data-view-component="true" class="Link--secondary Link">Security</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to status&quot;,&quot;label&quot;:&quot;text:status&quot;}" href="https://www.githubstatus.com/" data-view-component="true" class="Link--secondary Link">Status</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to community&quot;,&quot;label&quot;:&quot;text:community&quot;}" href="https://github.community/" data-view-component="true" class="Link--secondary Link">Community</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to docs&quot;,&quot;label&quot;:&quot;text:docs&quot;}" href="https://docs.github.com/" data-view-component="true" class="Link--secondary Link">Docs</a>
</li>
<li class="mx-2">
<a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to contact&quot;,&quot;label&quot;:&quot;text:contact&quot;}" href="https://support.github.com?tags=dotcom-footer" data-view-component="true" class="Link--secondary Link">Contact</a>
</li>
<li class="mx-2" >
<cookie-consent-link>
<button
type="button"
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent"
data-action="click:cookie-consent-link#showConsentManagement"
data-analytics-event="{&quot;location&quot;:&quot;footer&quot;,&quot;action&quot;:&quot;cookies&quot;,&quot;context&quot;:&quot;subfooter&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;cookies_link_subfooter_footer&quot;}"
>
Manage cookies
</button>
</cookie-consent-link>
</li>
<li class="mx-2">
<cookie-consent-link>
<button
type="button"
class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent text-left"
data-action="click:cookie-consent-link#showConsentManagement"
data-analytics-event="{&quot;location&quot;:&quot;footer&quot;,&quot;action&quot;:&quot;dont_share_info&quot;,&quot;context&quot;:&quot;subfooter&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;dont_share_info_link_subfooter_footer&quot;}"
>
Do not share my personal information
</button>
</cookie-consent-link>
</li>
</ul>
</nav>
</div>
</footer>
<ghcc-consent id="ghcc" class="position-fixed bottom-0 left-0" style="z-index: 999999"
data-locale="en"
data-initial-cookie-consent-allowed=""
data-cookie-consent-required="false"
></ghcc-consent>
<div id="ajax-error-message" class="ajax-error-message flash flash-error" hidden>
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
<path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
<button type="button" class="flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</button>
You can’t perform that action at this time.
</div>
<template id="site-details-dialog">
<details class="details-reset details-overlay details-overlay-dark lh-default color-fg-default hx_rsm" open>
<summary role="button" aria-label="Close dialog"></summary>
<details-dialog class="Box Box--overlay d-flex flex-column anim-fade-in fast hx_rsm-dialog hx_rsm-modal">
<button class="Box-btn-octicon m-0 btn-octicon position-absolute right-0 top-0" type="button" aria-label="Close dialog" data-close-dialog>
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</button>
<div class="octocat-spinner tmp-my-6 js-details-dialog-spinner"></div>
</details-dialog>
</details>
</template>
<div class="Popover js-hovercard-content position-absolute" style="display: none; outline: none;">
<div class="Popover-message Popover-message--bottom-left Popover-message--large Box color-shadow-large" style="width:360px;">
</div>
</div>
<template id="snippet-clipboard-copy-button">
<div class="zeroclipboard-container position-absolute right-0 top-0">
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn js-clipboard-copy m-2 p-0" data-copy-feedback="Copied!" data-tooltip-direction="w">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon m-2 tmp-m-2">
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none m-2 tmp-m-2">
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
</svg>
</clipboard-copy>
</div>
</template>
<template id="snippet-clipboard-copy-button-unpositioned">
<div class="zeroclipboard-container">
<clipboard-copy aria-label="Copy code to clipboard" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w">
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon">
<path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
<svg aria-hidden="true" data-component="Octicon" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none">
<path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
</svg>
</clipboard-copy>
</div>
</template>
</div>
<div id="js-global-screen-reader-notice" class="sr-only mt-n1" aria-live="polite" aria-atomic="true" ></div>
<div id="js-global-screen-reader-notice-assertive" class="sr-only mt-n1" aria-live="assertive" aria-atomic="true"></div>
</body>
</html>