Subject: Re: [PATCH] tcp: check socket state before calling WARN_ON

From: Youngmin Nam <[email-masked]>
Date: Mon, 3 Feb 2025 14:21:32 +0900

On Mon, Jan 20, 2025 at 09:18:48AM +0900, Youngmin Nam wrote:
> On Fri, Jan 17, 2025 at 10:18:58AM -0500, Neal Cardwell wrote:
> > On Fri, Jan 17, 2025 at 12:04 AM Youngmin Nam <[email-masked]> wrote:
> > >
> > > > Thanks for all the details! If the ramdump becomes available again at
> > > > some point, would it be possible to pull out the following values as
> > > > well:
> > > >
> > > > tp->mss_cache
> > > > inet_csk(sk)->icsk_pmtu_cookie
> > > > inet_csk(sk)->icsk_ca_state
> > > >
> > > > Thanks,
> > > > neal
> > > >
> > >
> > > Hi Neal. Happy new year.
> > >
> > > We are currently trying to capture a tcpdump during the problem situation
> > > to construct the Packetdrill script. However, this issue does not occur very often.
> > >
> > > By the way, we have a full ramdump, so we can provide the information you requested.
> > >
> > > tp->packets_out = 0
> > > tp->sacked_out = 0
> > > tp->lost_out = 4
> > > tp->retrans_out = 1
> > > tcp_is_sack(tp) = 1
> > > tp->mss_cache = 1428
> > > inet_csk(sk)->icsk_ca_state = 4
> > > inet_csk(sk)->icsk_pmtu_cookie = 1500
> > >
> > > If you need any specific information from the ramdump, please let me know.
> > 
> > The icsk_ca_state = 4 is interesting, since that's TCP_CA_Loss,
> > indicating RTO recovery. Perhaps the socket suffered many recurring
> > timeouts and timed out with ETIMEDOUT,
> > causing the tcp_write_queue_purge() call that reset packets_out to
> > 0... and then some race happened during the teardown process that
> > caused another incoming packet to be processed in this resulting
> > inconsistent state?
> > 
> > Do you have a way to use GDB or a similar tool to print all the fields
> > of the socket? Like:
> > 
> >   (gdb)  p *(struct tcp_sock*) some_hex_address_goes_here
> > 
> > ?
> > 
> > If so, that could be useful in extracting further hints about what
> > state this socket is in.
> > 
> > If that's not possible, but a few extra fields are possible, would you
> > be able to pull out the following:
> > 
> > tp->retrans_stamp
> > tp->tcp_mstamp
> > icsk->icsk_retransmits
> > icsk->icsk_backoff
> > icsk->icsk_rto
> > 
> > thanks,
> > neal
> > 
> 
> Hi Neal,
> Thank you for looking into this issue.
> When we first encountered this issue, we also suspected that tcp_write_queue_purge() was being called.
> We can provide any information you would like to inspect.
> 
> tp->retrans_stamp = 3339228
> tp->tcp_mstamp = 3552879949
> icsk->icsk_retransmits = 2
> icsk->icsk_backoff = 0
> icsk->icsk_rto = 16340
> 
> Here is all the information about tcp_sock.
> 
> (struct tcp_sock *) tp = 0xFFFFFF88C1053C00 -> (
>   (struct inet_connection_sock) inet_conn = ((struct inet_sock) icsk_inet = ((struct sock) sk = ((struct sock_common) __sk_common = ((__addrpair) skc_addrpair = 13979358786200921654, (__be32) skc_daddr = 255714870, (__be32) skc_rcv_saddr = 3254823104, (unsigned int) skc_hash = 2234333897, (__u16 [2]) skc_u16hashes = (15049, 34093), (__portpair) skc_portpair = 3600464641, (__be16) skc_dport = 47873, (__u16) skc_num = 54938, (unsigned short) skc_family = 10, (volatile unsigned char) skc_state = 4, (unsigned char:4) skc_reuse = 0, (unsigned char:1) skc_reuseport = 0, (unsigned char:1) skc_ipv6only = 0, (unsigned char:1) skc_net_refcnt = 1, (int) skc_bound_dev_if = 0, (struct hlist_node) skc_bind_node = ((struct hlist_node *) next = 0x0, (struct hlist_node * *) pprev = 0xFFFFFF881CE60AC0), (struct hlist_node) skc_portaddr_node = ((struct hlist_node *) next = 0x0, (struct hlist_node * *) pprev = 0xFFFFFF881CE60AC0), (struct proto *) skc_prot = 0xFFFFFFD08322CFE0, (possible_net_t) skc_net = ((struct net *) net = 0xFFFFFFD083316600), (struct in6_addr) skc_v6_daddr = ((union) in6_u = ((__u8 [16]) u6_addr8 = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 54, 230, 61, 15), (__be16 [8]) u6_addr16 = (0, 0, 0, 0, 0, 65535, 58934, 3901), (__be32 [4]) u6_addr32 = (0, 0, 4294901760, 255714870))), (struct in6_addr) skc_v6_rcv_saddr = ((union) in6_u = ((__u8 [16]) u6_addr8 = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 192, 168, 0, 194), (__be16 [8]) u6_addr16 = (0, 0, 0, 0, 0, 65535, 43200, 49664), (__be32 [4]) u6_addr32 = (0, 0, 4294901760, 3254823104))), (atomic64_t) skc_cookie = ((s64) counter = 6016), (unsigned long) skc_flags = 769, (struct sock *) skc_listener = 0x0301, (struct inet_timewait_death_row *) skc_tw_dr = 0x0301, (int [0]) skc_dontcopy_begin = (), (struct hlist_node) skc_node = ((struct hlist_node *) next = 0x7593, (struct hlist_node * *) pprev = 0xFFFFFF882D49D648), (struct hlist_nulls_node) skc_nulls_node = ((struct hlist_nulls_node *) next = 0x7593, (struct hlist_nulls_node * *) pprev = 0xFFFFFF882D49D648), (unsigned short) skc_tx_queue_mapping = 65535, (unsigned short) skc_rx_queue_mapping = 65535, (int) skc_incoming_cpu = 2, (u32) skc_rcv_wnd = 2, (u32) skc_tw_rcv_nxt = 2, (refcount_t) skc_refcnt = ((atomic_t) refs = ((int) counter = 2)), (int [0]) skc_dontcopy_end = (), (u32) skc_rxhash = 0, (u32) skc_window_clamp = 0, (u32) skc_tw_snd_nxt = 0), (struct dst_entry *) sk_rx_dst = 0xFFFFFF8821384F00, (int) sk_rx_dst_ifindex = 14, (u32) sk_rx_dst_cookie = 0, (socket_lock_t) sk_lock = ((spinlock_t) slock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 1), (u8) locked = 1, (u8) pending = 0, (u16) locked_pending = 1, (u16) tail = 0))), (int) owned = 0, (wait_queue_head_t) wq = ((spinlock_t) lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_loc
>   (u16) tcp_header_len = 32,
>   (u16) gso_segs = 26,
>   (__be32) pred_flags = 2566918272,
>   (u64) bytes_received = 7950,
>   (u32) segs_in = 21,
>   (u32) data_segs_in = 12,
>   (u32) rcv_nxt = 2956372822,
>   (u32) copied_seq = 2956372822,
>   (u32) rcv_wup = 2956372822,
>   (u32) snd_nxt = 2253384964,
>   (u32) segs_out = 30,
>   (u32) data_segs_out = 13,
>   (u64) bytes_sent = 11381,
>   (u64) bytes_acked = 3841,
>   (u32) dsack_dups = 0,
>   (u32) snd_una = 2253381091,
>   (u32) snd_sml = 2253384964,
>   (u32) rcv_tstamp = 757520,
>   (u32) lsndtime = 768000,
>   (u32) last_oow_ack_time = 750065,
>   (u32) compressed_ack_rcv_nxt = 2956370839,
>   (u32) tsoffset = 1072186408,
>   (struct list_head) tsq_node = ((struct list_head *) next = 0xFFFFFF88C1054260, (struct list_head *) prev = 0xFFFFFF88C1054260),
>   (struct list_head) tsorted_sent_queue = ((struct list_head *) next = 0xFFFFFF88C1054270, (struct list_head *) prev = 0xFFFFFF88C1054270),
>   (u32) snd_wl1 = 2956372861,
>   (u32) snd_wnd = 78336,
>   (u32) max_window = 78336,
>   (u32) mss_cache = 1428,
>   (u32) window_clamp = 2752512,
>   (u32) rcv_ssthresh = 91439,
>   (u8) scaling_ratio = 84,
>   (struct tcp_rack) rack = ((u64) mstamp = 3312043553, (u32) rtt_us = 3103952, (u32) end_seq = 2253381091, (u32) last_delivered = 0, (u8) reo_wnd_steps = 1, (u8:5) reo_wnd_persist = 0, (u8:1) dsack_seen = 0, (u8:1) advanced = 1),
>   (u16) advmss = 1448,
>   (u8) compressed_ack = 0,
>   (u8:2) dup_ack_counter = 2,
>   (u8:1) tlp_retrans = 1,
>   (u8:5) unused = 0,
>   (u32) chrono_start = 780636,
>   (u32 [3]) chrono_stat = (30386, 0, 0),
>   (u8:2) chrono_type = 1,
>   (u8:1) rate_app_limited = 1,
>   (u8:1) fastopen_connect = 0,
>   (u8:1) fastopen_no_cookie = 0,
>   (u8:1) is_sack_reneg = 0,
>   (u8:2) fastopen_client_fail = 0,
>   (u8:4) nonagle = 0,
>   (u8:1) thin_lto = 0,
>   (u8:1) recvmsg_inq = 0,
>   (u8:1) repair = 0,
>   (u8:1) frto = 1,
>   (u8) repair_queue = 0,
>   (u8:2) save_syn = 0,
>   (u8:1) syn_data = 0,
>   (u8:1) syn_fastopen = 0,
>   (u8:1) syn_fastopen_exp = 0,
>   (u8:1) syn_fastopen_ch = 0,
>   (u8:1) syn_data_acked = 0,
>   (u8:1) is_cwnd_limited = 1,
>   (u32) tlp_high_seq = 0,
>   (u32) tcp_tx_delay = 0,
>   (u64) tcp_wstamp_ns = 3371996070858,
>   (u64) tcp_clock_cache = 3552879949296,
>   (u64) tcp_mstamp = 3552879949,
>   (u32) srtt_us = 29633751,
>   (u32) mdev_us = 10160190,
>   (u32) mdev_max_us = 10160190,
>   (u32) rttvar_us = 12632227,
>   (u32) rtt_seq = 2253383947,
>   (struct minmax) rtt_min = ((struct minmax_sample [3]) s = (((u32) t = 753091, (u32) v = 330326), ((u32) t = 753091, (u32) v = 330326), ((u32) t = 753091, (u32) v = 330326))),
>   (u32) packets_out = 0,
>   (u32) retrans_out = 1,
>   (u32) max_packets_out = 4,
>   (u32) cwnd_usage_seq = 2253384964,
>   (u16) urg_data = 0,
>   (u8) ecn_flags = 0,
>   (u8) keepalive_probes = 0,
>   (u32) reordering = 3,
>   (u32) reord_seen = 0,
>   (u32) snd_up = 2253381091,
>   (struct tcp_options_received) rx_opt = ((int) ts_recent_stamp = 3330, (u32) ts_recent = 1119503967, (u32) rcv_tsval = 1119728668, (u32) rcv_tsecr = 3312043, (u16:1) saw_tstamp = 1, (u16:1) tstamp_ok = 1, (u16:1) dsack = 0, (u16:1) wscale_ok = 1, (u16:3) sack_ok = 1, (u16:1) smc_ok = 0, (u16:4) snd_wscale = 9, (u16:4) rcv_wscale = 9, (u8:1) saw_unknown = 0, (u8:7) unused = 0, (u8) num_sacks = 0, (u16) user_mss = 0, (u16) mss_clamp = 1440),
>   (u32) snd_ssthresh = 7,
>   (u32) snd_cwnd = 1,
>   (u32) snd_cwnd_cnt = 0,
>   (u32) snd_cwnd_clamp = 4294967295,
>   (u32) snd_cwnd_used = 0,
>   (u32) snd_cwnd_stamp = 768000,
>   (u32) prior_cwnd = 10,
>   (u32) prr_delivered = 0,
>   (u32) prr_out = 0,
>   (u32) delivered = 7,
>   (u32) delivered_ce = 0,
>   (u32) lost = 8,
>   (u32) app_limited = 8,
>   (u64) first_tx_mstamp = 3312043553,
>   (u64) delivered_mstamp = 3315147505,
>   (u32) rate_delivered = 1,
>   (u32) rate_interval_us = 330326,
>   (u32) rcv_wnd = 91648,
>   (u32) write_seq = 2253384964,
>   (u32) notsent_lowat = 0,
>   (u32) pushed_seq = 2253384963,
>   (u32) lost_out = 4,
>   (u32) sacked_out = 0,
>   (struct hrtimer) pacing_timer = ((struct timerqueue_node) node = ((struct rb_node) node = ((unsigned long) __rb_parent_color = 18446743561551823800, (struct rb_node *) rb_right = 0x0, (struct rb_node *) rb_left = 0x0), (ktime_t) expires = 0), (ktime_t) _softexpires = 0, (enum hrtimer_restart (*)()) function = 0xFFFFFFD081EA565C, (struct hrtimer_clock_base *) base = 0xFFFFFF8962589DC0, (u8) state = 0, (u8) is_rel = 0, (u8) is_soft = 1, (u8) is_hard = 0, (u64) android_kabi_reserved1 = 0),
>   (struct hrtimer) compressed_ack_timer = ((struct timerqueue_node) node = ((struct rb_node) node = ((unsigned long) __rb_parent_color = 18446743561551823872, (struct rb_node *) rb_right = 0x0, (struct rb_node *) rb_left = 0x0), (ktime_t) expires = 0), (ktime_t) _softexpires = 0, (enum hrtimer_restart (*)()) function = 0xFFFFFFD081EAE348, (struct hrtimer_clock_base *) base = 0xFFFFFF8962589DC0, (u8) state = 0, (u8) is_rel = 0, (u8) is_soft = 1, (u8) is_hard = 0, (u64) android_kabi_reserved1 = 0),
>   (struct sk_buff *) lost_skb_hint = 0x0,
>   (struct sk_buff *) retransmit_skb_hint = 0x0,
>   (struct rb_root) out_of_order_queue = ((struct rb_node *) rb_node = 0x0),
>   (struct sk_buff *) ooo_last_skb = 0xFFFFFF891768FB00,
>   (struct tcp_sack_block [1]) duplicate_sack = (((u32) start_seq = 2956372143, (u32) end_seq = 2956372822)),
>   (struct tcp_sack_block [4]) selective_acks = (((u32) start_seq = 2956371078, (u32) end_seq = 2956372143), ((u32) start_seq = 0, (u32) end_seq = 0), ((u32) start_seq = 0, (u32) end_seq = 0), ((u32) start_seq = 0, (u32) end_seq = 0)),
>   (struct tcp_sack_block [4]) recv_sack_cache = (((u32) start_seq = 0, (u32) end_seq = 0), ((u32) start_seq = 0, (u32) end_seq = 0), ((u32) start_seq = 0, (u32) end_seq = 0), ((u32) start_seq = 0, (u32) end_seq = 0)),
>   (struct sk_buff *) highest_sack = 0x0,
>   (int) lost_cnt_hint = 0,
>   (u32) prior_ssthresh = 2147483647,
>   (u32) high_seq = 2253384964,
>   (u32) retrans_stamp = 3339228,
>   (u32) undo_marker = 2253381091,
>   (int) undo_retrans = 3,
>   (u64) bytes_retrans = 3669,
>   (u32) total_retrans = 6,
>   (u32) urg_seq = 0,
>   (unsigned int) keepalive_time = 0,
>   (unsigned int) keepalive_intvl = 0,
>   (int) linger2 = 0,
>   (u8) bpf_sock_ops_cb_flags = 0,
>   (u8:1) bpf_chg_cc_inprogress = 0,
>   (u16) timeout_rehash = 5,
>   (u32) rcv_ooopack = 2,
>   (u32) rcv_rtt_last_tsecr = 3312043,
>   (struct) rcv_rtt_est = ((u32) rtt_us = 0, (u32) seq = 2956431836, (u64) time = 3301136665),
>   (struct) rcvq_space = ((u32) space = 14480, (u32) seq = 2956364872, (u64) time = 3300257291),
>   (struct) mtu_probe = ((u32) probe_seq_start = 0, (u32) probe_seq_end = 0),
>   (u32) plb_rehash = 0,
>   (u32) mtu_info = 0,
>   (struct tcp_fastopen_request *) fastopen_req = 0x0,
>   (struct request_sock *) fastopen_rsk = 0x0,
>   (struct saved_syn *) saved_syn = 0x0,
>   (u64) android_oem_data1 = 0,
>   (u64) android_kabi_reserved1 = 0)
> 
> And here are the details of inet_connection_sock.
> 
> (struct tcp_sock *) tp = 0xFFFFFF88C1053C00 -> (
>   (struct inet_connection_sock) inet_conn = (
>     (struct inet_sock) icsk_inet = ((struct sock) sk = ((struct sock_common) __sk_common = ((__addrpair) skc_addrpair = 13979358786200921654, (__be32) skc_daddr = 255714870, (__be32) skc_rcv_saddr = 3254823104, (unsigned int) skc_hash = 2234333897, (__u16 [2]) skc_u16hashes = (15049, 34093), (__portpair) skc_portpair = 3600464641, (__be16) skc_dport = 47873, (__u16) skc_num = 54938, (unsigned short) skc_family = 10, (volatile unsigned char) skc_state = 4, (unsigned char:4) skc_reuse = 0, (unsigned char:1) skc_reuseport = 0, (unsigned char:1) skc_ipv6only = 0, (unsigned char:1) skc_net_refcnt = 1, (int) skc_bound_dev_if = 0, (struct hlist_node) skc_bind_node = ((struct hlist_node *) next = 0x0, (struct hlist_node * *) pprev = 0xFFFFFF881CE60AC0), (struct hlist_node) skc_portaddr_node = ((struct hlist_node *) next = 0x0, (struct hlist_node * *) pprev = 0xFFFFFF881CE60AC0), (struct proto *) skc_prot = 0xFFFFFFD08322CFE0, (possible_net_t) skc_net = ((struct net *) net = 0xFFFFFFD083316600), (struct in6_addr) skc_v6_daddr = ((union) in6_u = ((__u8 [16]) u6_addr8 = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 54, 230, 61, 15), (__be16 [8]) u6_addr16 = (0, 0, 0, 0, 0, 65535, 58934, 3901), (__be32 [4]) u6_addr32 = (0, 0, 4294901760, 255714870))), (struct in6_addr) skc_v6_rcv_saddr = ((union) in6_u = ((__u8 [16]) u6_addr8 = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 192, 168, 0, 194), (__be16 [8]) u6_addr16 = (0, 0, 0, 0, 0, 65535, 43200, 49664), (__be32 [4]) u6_addr32 = (0, 0, 4294901760, 3254823104))), (atomic64_t) skc_cookie = ((s64) counter = 6016), (unsigned long) skc_flags = 769, (struct sock *) skc_listener = 0x0301, (struct inet_timewait_death_row *) skc_tw_dr = 0x0301, (int [0]) skc_dontcopy_begin = (), (struct hlist_node) skc_node = ((struct hlist_node *) next = 0x7593, (struct hlist_node * *) pprev = 0xFFFFFF882D49D648), (struct hlist_nulls_node) skc_nulls_node = ((struct hlist_nulls_node *) next = 0x7593, (struct hlist_nulls_node * *) pprev = 0xFFFFFF882D49D648), (unsigned short) skc_tx_queue_mapping = 65535, (unsigned short) skc_rx_queue_mapping = 65535, (int) skc_incoming_cpu = 2, (u32) skc_rcv_wnd = 2, (u32) skc_tw_rcv_nxt = 2, (refcount_t) skc_refcnt = ((atomic_t) refs = ((int) counter = 2)), (int [0]) skc_dontcopy_end = (), (u32) skc_rxhash = 0, (u32) skc_window_clamp = 0, (u32) skc_tw_snd_nxt = 0), (struct dst_entry *) sk_rx_dst = 0xFFFFFF8821384F00, (int) sk_rx_dst_ifindex = 14, (u32) sk_rx_dst_cookie = 0, (socket_lock_t) sk_lock = ((spinlock_t) slock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 1), (u8) locked = 1, (u8) pending = 0, (u16) locked_pending = 1, (u16) tail = 0))), (int) owned = 0, (wait_queue_head_t) wq = ((spinlock_t) lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 0), (u8) locked = 0, (u8) pending = 0, (u16) locked_pending = 0, (u16) tail = 0))), (struct list_head) head = ((struct list_head *) next = 0xFFFFFF88C1053CA8, (struct list_head *) prev = 0xFFFFFF88C1053CA8))), (atomic_t) sk_drops = ((int) counter = 7), (int) sk_rcvlowat = 1, (struct sk_buff_head) sk_error_queue = ((struct sk_buff *) next = 0xFFFFFF88C1053CC0, (struct sk_buff *) prev = 0xFFFFFF88C1053CC0, (struct sk_buff_list) list = ((struct sk_buff *) next = 0xFFFFFF88C1053CC0, (struct sk_buff *) prev = 0xFFFFFF88C1053CC0), (__u32) qlen = 0, (spinlock_t) lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 0), (u8) locked = 0, (u8) pending = 0, (u16) locked_pending = 0, (u16) tail = 0)))), (struct sk_buff_head) sk_receive_queue = ((struct sk_buff *) next = 0xFFFFFF88C1053CD8, (struct sk_buff *) prev = 0xFFFFFF88C1053CD8, (struct sk_buff_list) list = ((struct sk_buff *) next = 0xFFFFFF88C1053CD8, (struct sk_buff *) prev = 0xFFFFFF88C1053CD8), (__u32) qlen = 0, (spinlock_t) lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 0), (u8) locked = 0, (u8) pending = 0, (u16) locked_pending = 0, (u16) tail = 0)))), (struct) sk_backlog = ((at
>     (struct request_sock_queue) icsk_accept_queue = ((spinlock_t) rskq_lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 0), (u8) locked = 0, (u8) pending = 0, (u16) locked_pending = 0, (u16) tail = 0))), (u8) rskq_defer_accept = 0, (u32) synflood_warned = 0, (atomic_t) qlen = ((int) counter = 0), (atomic_t) young = ((int) counter = 0), (struct request_sock *) rskq_accept_head = 0x0, (struct request_sock *) rskq_accept_tail = 0x0, (struct fastopen_queue) fastopenq = ((struct request_sock *) rskq_rst_head = 0x0, (struct request_sock *) rskq_rst_tail = 0x0, (spinlock_t) lock = ((struct raw_spinlock) rlock = ((arch_spinlock_t) raw_lock = ((atomic_t) val = ((int) counter = 0), (u8) locked = 0, (u8) pending = 0, (u16) locked_pending = 0, (u16) tail = 0))), (int) qlen = 0, (int) max_qlen = 0, (struct tcp_fastopen_context *) ctx = 0x0)),
>     (struct inet_bind_bucket *) icsk_bind_hash = 0xFFFFFF881CE60A80,
>     (struct inet_bind2_bucket *) icsk_bind2_hash = 0xFFFFFF881B1FD500,
>     (unsigned long) icsk_timeout = 4295751636,
>     (struct timer_list) icsk_retransmit_timer = ((struct hlist_node) entry = ((struct hlist_node *) next = 0xDEAD000000000122, (struct hlist_node * *) pprev = 0x0), (unsigned long) expires = 4295751636, (void (*)()) function = 0xFFFFFFD081EADE28, (u32) flags = 1056964613, (u64) android_kabi_reserved1 = 0, (u64) android_kabi_reserved2 = 0),
>     (struct timer_list) icsk_delack_timer = ((struct hlist_node) entry = ((struct hlist_node *) next = 0xDEAD000000000122, (struct hlist_node * *) pprev = 0x0), (unsigned long) expires = 4295721093, (void (*)()) function = 0xFFFFFFD081EADF60, (u32) flags = 25165829, (u64) android_kabi_reserved1 = 0, (u64) android_kabi_reserved2 = 0),
>     (__u32) icsk_rto_=_16340,
>     (__u32) icsk_rto_min = 50,
>     (__u32) icsk_delack_max = 50,
>     (__u32) icsk_pmtu_cookie = 1500,
>     (const struct tcp_congestion_ops *) icsk_ca_ops = 0xFFFFFFD0830A9440,
>     (const struct inet_connection_sock_af_ops *) icsk_af_ops = 0xFFFFFFD082290FC8,
>     (const struct tcp_ulp_ops *) icsk_ulp_ops = 0x0,
>     (void *) icsk_ulp_data = 0x0,
>     (void (*)()) icsk_clean_acked = 0x0,
>     (unsigned int (*)()) icsk_sync_mss = 0xFFFFFFD081EA60EC,
>     (__u8:5) icsk_ca_state = 4,
>     (__u8:1) icsk_ca_initialized = 1,
>     (__u8:1) icsk_ca_setsockopt = 0,
>     (__u8:1) icsk_ca_dst_locked = 0,
>     (__u8) icsk_retransmits = 2,
>     (__u8) icsk_pending = 0,
>     (__u8) icsk_backoff = 0,
>     (__u8) icsk_syn_retries = 0,
>     (__u8) icsk_probes_out = 0,
>     (__u16) icsk_ext_hdr_len = 0,
>     (struct) icsk_ack = ((__u8) pending = 0, (__u8) quick = 15, (__u8) pingpong = 0, (__u8) retry = 0, (__u32) ato = 10, (unsigned long) timeout = 4295721093, (__u32) lrcvtime = 753787, (__u16) last_seg_size = 0, (__u16) rcv_mss = 1428),
>     (struct) icsk_mtup = ((int) search_high = 1480, (int) search_low = 1076, (u32:31) probe_size = 0, (u32:1) enabled = 0, (u32) probe_timestamp = 0),
>     (u32) icsk_probes_tstamp = 0,
>     (u32) icsk_user_timeout = 0,
>     (u64) android_kabi_reserved1 = 0,
>     (u64 [13]) icsk_ca_priv = (0, 0, 0, 0, 0, 14482612842890526720, 14482612845143911684, 4294967295, 0, 0, 0, 0, 0)),

Hi Neal.

When you have a chance, could you take a look at this ramdump snapshot?