Lines Matching refs:ssk
316 static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size) in mptcp_rmem_schedule() argument
328 if (ssk->sk_forward_alloc < amount) { in mptcp_rmem_schedule()
333 ssk->sk_forward_alloc -= amount; in mptcp_rmem_schedule()
338 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
342 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skb()
347 __skb_unlink(skb, &ssk->sk_receive_queue); in __mptcp_move_skb()
353 if (!mptcp_rmem_schedule(sk, ssk, skb->truesize)) in __mptcp_move_skb()
479 const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_timeout_from_subflow() local
481 return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? in mptcp_timeout_from_subflow()
482 inet_csk(ssk)->icsk_timeout - jiffies : 0; in mptcp_timeout_from_subflow()
495 static bool tcp_can_send_ack(const struct sock *ssk) in tcp_can_send_ack() argument
497 return !((1 << inet_sk_state_load(ssk)) & in tcp_can_send_ack()
501 void mptcp_subflow_send_ack(struct sock *ssk) in mptcp_subflow_send_ack() argument
505 slow = lock_sock_fast(ssk); in mptcp_subflow_send_ack()
506 if (tcp_can_send_ack(ssk)) in mptcp_subflow_send_ack()
507 tcp_send_ack(ssk); in mptcp_subflow_send_ack()
508 unlock_sock_fast(ssk, slow); in mptcp_subflow_send_ack()
519 static void mptcp_subflow_cleanup_rbuf(struct sock *ssk) in mptcp_subflow_cleanup_rbuf() argument
523 slow = lock_sock_fast(ssk); in mptcp_subflow_cleanup_rbuf()
524 if (tcp_can_send_ack(ssk)) in mptcp_subflow_cleanup_rbuf()
525 tcp_cleanup_rbuf(ssk, 1); in mptcp_subflow_cleanup_rbuf()
526 unlock_sock_fast(ssk, slow); in mptcp_subflow_cleanup_rbuf()
529 static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) in mptcp_subflow_could_cleanup() argument
531 const struct inet_connection_sock *icsk = inet_csk(ssk); in mptcp_subflow_could_cleanup()
533 const struct tcp_sock *tp = tcp_sk(ssk); in mptcp_subflow_could_cleanup()
554 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_cleanup_rbuf() local
556 if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) in mptcp_cleanup_rbuf()
557 mptcp_subflow_cleanup_rbuf(ssk); in mptcp_cleanup_rbuf()
614 struct sock *ssk, in __mptcp_move_skbs_from_subflow() argument
617 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skbs_from_subflow()
628 int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); in __mptcp_move_skbs_from_subflow()
636 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
637 tp = tcp_sk(ssk); in __mptcp_move_skbs_from_subflow()
648 skb = skb_peek(&ssk->sk_receive_queue); in __mptcp_move_skbs_from_subflow()
681 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
689 sk_eat_skb(ssk, skb); in __mptcp_move_skbs_from_subflow()
694 more_data_avail = mptcp_subflow_data_available(ssk); in __mptcp_move_skbs_from_subflow()
752 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
757 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
759 if (unlikely(ssk->sk_err)) { in move_skbs_to_msk()
776 void mptcp_data_ready(struct sock *sk, struct sock *ssk) in mptcp_data_ready() argument
778 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_data_ready()
789 ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); in mptcp_data_ready()
802 if (move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
1093 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_enter_memory_pressure() local
1096 tcp_enter_memory_pressure(ssk); in mptcp_enter_memory_pressure()
1097 sk_stream_moderate_sndbuf(ssk); in mptcp_enter_memory_pressure()
1188 static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) in __mptcp_alloc_tx_skb() argument
1196 if (likely(sk_wmem_schedule(ssk, skb->truesize))) { in __mptcp_alloc_tx_skb()
1197 tcp_skb_entail(ssk, skb); in __mptcp_alloc_tx_skb()
1204 static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) in mptcp_alloc_tx_skb() argument
1214 return __mptcp_alloc_tx_skb(sk, ssk, gfp); in mptcp_alloc_tx_skb()
1229 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, in mptcp_sendmsg_frag() argument
1245 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); in mptcp_sendmsg_frag()
1252 info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); in mptcp_sendmsg_frag()
1255 skb = tcp_write_queue_tail(ssk); in mptcp_sendmsg_frag()
1272 tcp_mark_push(tcp_sk(ssk), skb); in mptcp_sendmsg_frag()
1279 skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); in mptcp_sendmsg_frag()
1294 tcp_remove_empty_skb(ssk); in mptcp_sendmsg_frag()
1309 if (!sk_wmem_schedule(ssk, copy)) { in mptcp_sendmsg_frag()
1310 tcp_remove_empty_skb(ssk); in mptcp_sendmsg_frag()
1324 sk_wmem_queued_add(ssk, copy); in mptcp_sendmsg_frag()
1325 sk_mem_charge(ssk, copy); in mptcp_sendmsg_frag()
1326 WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); in mptcp_sendmsg_frag()
1340 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; in mptcp_sendmsg_frag()
1350 mptcp_subflow_ctx(ssk)->rel_write_seq += copy; in mptcp_sendmsg_frag()
1354 tcp_push_pending_frames(ssk); in mptcp_sendmsg_frag()
1360 mptcp_subflow_ctx(ssk)->rel_write_seq += copy; in mptcp_sendmsg_frag()
1371 struct sock *ssk; member
1407 struct sock *ssk; in mptcp_subflow_get_send() local
1430 send_info[i].ssk = NULL; in mptcp_subflow_get_send()
1435 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_send()
1441 if (!sk_stream_memory_free(subflow->tcp_sock) || !tcp_sk(ssk)->snd_wnd) in mptcp_subflow_get_send()
1444 pace = READ_ONCE(ssk->sk_pacing_rate); in mptcp_subflow_get_send()
1448 ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, in mptcp_subflow_get_send()
1451 send_info[subflow->backup].ssk = ssk; in mptcp_subflow_get_send()
1459 send_info[0].ssk = send_info[1].ssk; in mptcp_subflow_get_send()
1461 if (send_info[0].ssk) { in mptcp_subflow_get_send()
1462 msk->last_snd = send_info[0].ssk; in mptcp_subflow_get_send()
1471 static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) in mptcp_push_release() argument
1473 tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); in mptcp_push_release()
1474 release_sock(ssk); in mptcp_push_release()
1511 struct sock *prev_ssk = NULL, *ssk = NULL; in __mptcp_push_pending() local
1526 prev_ssk = ssk; in __mptcp_push_pending()
1528 ssk = mptcp_subflow_get_send(msk); in __mptcp_push_pending()
1533 if (ssk != prev_ssk && prev_ssk) in __mptcp_push_pending()
1535 if (!ssk) in __mptcp_push_pending()
1542 if (ssk != prev_ssk) in __mptcp_push_pending()
1543 lock_sock(ssk); in __mptcp_push_pending()
1545 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); in __mptcp_push_pending()
1547 mptcp_push_release(ssk, &info); in __mptcp_push_pending()
1561 if (ssk) in __mptcp_push_pending()
1562 mptcp_push_release(ssk, &info); in __mptcp_push_pending()
1572 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk) in __mptcp_subflow_push_pending() argument
1595 xmit_ssk = first ? ssk : mptcp_subflow_get_send(mptcp_sk(sk)); in __mptcp_subflow_push_pending()
1598 if (xmit_ssk != ssk) { in __mptcp_subflow_push_pending()
1604 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); in __mptcp_subflow_push_pending()
1623 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, in __mptcp_subflow_push_pending()
1886 struct sock *ssk; in mptcp_rcv_space_adjust() local
1889 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_rcv_space_adjust()
1890 slow = lock_sock_fast(ssk); in mptcp_rcv_space_adjust()
1891 WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); in mptcp_rcv_space_adjust()
1892 tcp_sk(ssk)->window_clamp = window_clamp; in mptcp_rcv_space_adjust()
1893 tcp_cleanup_rbuf(ssk, 1); in mptcp_rcv_space_adjust()
1894 unlock_sock_fast(ssk, slow); in mptcp_rcv_space_adjust()
1932 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs() local
1939 if (likely(!ssk)) in __mptcp_move_skbs()
1942 slowpath = lock_sock_fast(ssk); in __mptcp_move_skbs()
1945 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
1948 if (unlikely(ssk->sk_err)) in __mptcp_move_skbs()
1950 unlock_sock_fast(ssk, slowpath); in __mptcp_move_skbs()
2122 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_retrans() local
2128 if (!tcp_rtx_and_write_queues_empty(ssk)) { in mptcp_subflow_get_retrans()
2129 mptcp_pm_subflow_chk_stale(msk, ssk); in mptcp_subflow_get_retrans()
2136 backup = ssk; in mptcp_subflow_get_retrans()
2141 pick = ssk; in mptcp_subflow_get_retrans()
2207 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, in __mptcp_close_ssk() argument
2215 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); in __mptcp_close_ssk()
2220 if (ssk->sk_socket) in __mptcp_close_ssk()
2221 sock_orphan(ssk); in __mptcp_close_ssk()
2230 if (!inet_csk(ssk)->icsk_ulp_ops) { in __mptcp_close_ssk()
2234 __tcp_close(ssk, 0); in __mptcp_close_ssk()
2237 __sock_put(ssk); in __mptcp_close_ssk()
2239 release_sock(ssk); in __mptcp_close_ssk()
2241 sock_put(ssk); in __mptcp_close_ssk()
2243 if (ssk == msk->last_snd) in __mptcp_close_ssk()
2246 if (ssk == msk->first) in __mptcp_close_ssk()
2249 if (msk->subflow && ssk == msk->subflow->sk) in __mptcp_close_ssk()
2256 void mptcp_close_ssk(struct sock *sk, struct sock *ssk, in mptcp_close_ssk() argument
2260 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); in mptcp_close_ssk()
2261 __mptcp_close_ssk(sk, ssk, subflow); in mptcp_close_ssk()
2276 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_close_subflow() local
2278 if (inet_sk_state_load(ssk) != TCP_CLOSE) in __mptcp_close_subflow()
2282 if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) in __mptcp_close_subflow()
2285 mptcp_close_ssk((struct sock *)msk, ssk, subflow); in __mptcp_close_subflow()
2344 struct sock *ssk; in __mptcp_retrans() local
2350 ssk = mptcp_subflow_get_retrans(msk); in __mptcp_retrans()
2369 if (!ssk) in __mptcp_retrans()
2372 lock_sock(ssk); in __mptcp_retrans()
2378 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); in __mptcp_retrans()
2388 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, in __mptcp_retrans()
2392 release_sock(ssk); in __mptcp_retrans()
2534 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) in mptcp_subflow_shutdown() argument
2536 lock_sock(ssk); in mptcp_subflow_shutdown()
2538 switch (ssk->sk_state) { in mptcp_subflow_shutdown()
2544 tcp_disconnect(ssk, O_NONBLOCK); in mptcp_subflow_shutdown()
2549 ssk->sk_shutdown |= how; in mptcp_subflow_shutdown()
2550 tcp_shutdown(ssk, how); in mptcp_subflow_shutdown()
2552 pr_debug("Sending DATA_FIN on subflow %p", ssk); in mptcp_subflow_shutdown()
2553 tcp_send_ack(ssk); in mptcp_subflow_shutdown()
2560 release_sock(ssk); in mptcp_subflow_shutdown()
2666 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_destroy_sock() local
2667 __mptcp_close_ssk(sk, ssk, subflow); in __mptcp_destroy_sock()
2704 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_close() local
2705 bool slow = lock_sock_fast_nested(ssk); in mptcp_close()
2707 sock_orphan(ssk); in mptcp_close()
2708 unlock_sock_fast(ssk, slow); in mptcp_close()
2730 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
2733 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); in mptcp_copy_inaddrs()
2736 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
2737 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
2745 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
2746 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
2747 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
2748 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
2749 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
2750 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
2761 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_disconnect() local
2763 lock_sock(ssk); in mptcp_disconnect()
2764 tcp_disconnect(ssk, flags); in mptcp_disconnect()
2765 release_sock(ssk); in mptcp_disconnect()
2833 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
2835 const struct tcp_sock *tp = tcp_sk(ssk); in mptcp_rcv_space_init()
2848 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); in mptcp_rcv_space_init()
2938 void __mptcp_check_push(struct sock *sk, struct sock *ssk) in __mptcp_check_push() argument
2946 if (xmit_ssk == ssk) in __mptcp_check_push()
2947 __mptcp_subflow_push_pending(sk, ssk); in __mptcp_check_push()
3003 static void schedule_3rdack_retransmission(struct sock *ssk) in schedule_3rdack_retransmission() argument
3005 struct inet_connection_sock *icsk = inet_csk(ssk); in schedule_3rdack_retransmission()
3006 struct tcp_sock *tp = tcp_sk(ssk); in schedule_3rdack_retransmission()
3009 if (mptcp_subflow_ctx(ssk)->fully_established) in schedule_3rdack_retransmission()
3022 sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); in schedule_3rdack_retransmission()
3025 void mptcp_subflow_process_delegated(struct sock *ssk) in mptcp_subflow_process_delegated() argument
3027 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_subflow_process_delegated()
3033 __mptcp_subflow_push_pending(sk, ssk); in mptcp_subflow_process_delegated()
3040 schedule_3rdack_retransmission(ssk); in mptcp_subflow_process_delegated()
3072 void mptcp_finish_connect(struct sock *ssk) in mptcp_finish_connect() argument
3079 subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_connect()
3102 mptcp_pm_new_connection(msk, ssk, 0); in mptcp_finish_connect()
3104 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
3116 bool mptcp_finish_join(struct sock *ssk) in mptcp_finish_join() argument
3118 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_join()
3149 sock_hold(ssk); in mptcp_finish_join()
3161 if (parent_sock && !ssk->sk_socket) in mptcp_finish_join()
3162 mptcp_sock_graft(ssk, parent_sock); in mptcp_finish_join()
3165 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); in mptcp_finish_join()
3377 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_stream_accept() local
3379 if (!ssk->sk_socket) in mptcp_stream_accept()
3380 mptcp_sock_graft(ssk, newsock); in mptcp_stream_accept()
3496 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_napi_poll() local
3498 bh_lock_sock_nested(ssk); in mptcp_napi_poll()
3499 if (!sock_owned_by_user(ssk) && in mptcp_napi_poll()
3501 mptcp_subflow_process_delegated(ssk); in mptcp_napi_poll()
3507 bh_unlock_sock(ssk); in mptcp_napi_poll()
3508 sock_put(ssk); in mptcp_napi_poll()