From 54075ba4f00ca770d5d158805f30116b0297c14e Mon Sep 17 00:00:00 2001 From: Tony Lu Date: Wed, 24 Feb 2021 09:18:20 +0800 Subject: [PATCH 2553/2944] alinux: tcp: introduce tunable tcp_rto_min value fix #33086900 TCP_RTO_MIN is fixed to 200ms which is a little long period for latency-sensitive applications deployed in the same data center. This introduces a brand new sysctl knob net.ipv4.tcp_rto_min to tune the default minimum TCP RTO per net namespace, and the default value is still 200ms. Also a new TCP socket option TCP_RTO_MINIMAL is added. The socket option of TCP_RTO_MINIMAL has a higher priority than sysctl, which means TCP connection would use the socket-level minimal RTO if setted, or use the namespace-level value. And socket option value < 1 means not setted. Besides that, if the provided value is smaller than one tick, use one tick instead. TCP_MIB_RTOMIN is equal to the sysctl net.ipv4.tcp_rto_min of current net namespace. Setting the TCP_RTO_MINIMAL of TCP socket option won't update TCP_MIB_RTOMIN. Signed-off-by: Tony Lu Acked-by: Dust Li --- Documentation/networking/ip-sysctl.txt | 6 ++++ include/linux/sysctl.h | 3 ++ include/linux/tcp.h | 2 ++ include/net/netns/ipv4.h | 1 + include/net/snmp.h | 8 +++++ include/net/tcp.h | 6 ++-- include/uapi/linux/tcp.h | 1 + kernel/sysctl.c | 58 ++++++++++++++++++++++++++++++++++ net/ipv4/sysctl_net_ipv4.c | 29 +++++++++++++++++ net/ipv4/tcp.c | 9 ++++++ net/ipv4/tcp_ipv4.c | 1 + net/ipv4/tcp_output.c | 5 ++- net/ipv4/tcp_timer.c | 8 ++++- 13 files changed, 133 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index e26d94e..f4f9ae2 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -774,6 +774,12 @@ tcp_challenge_ack_limit - INTEGER in RFC 5961 (Improving TCP's Robustness to Blind In-Window Attacks) Default: 100 +tcp_rto_min - INTEGER + The minminal timeout (in milliseconds) before the TCP segments are + retransmitted. This value is per net namespace and not inherited. + If the value is smaller than one tick, use one tick. + Default: 200 ms + UDP variables: udp_l3mdev_accept - BOOLEAN diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index d36e15c..9e46096 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -59,6 +59,9 @@ extern int proc_dointvec_userhz_jiffies(struct ctl_table *, int, void __user *, size_t *, loff_t *); extern int proc_dointvec_ms_jiffies(struct ctl_table *, int, void __user *, size_t *, loff_t *); +extern int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); extern int proc_doulongvec_minmax(struct ctl_table *, int, void __user *, size_t *, loff_t *); extern int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int, diff --git a/include/linux/tcp.h b/include/linux/tcp.h index c615459..e2da0ca 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -400,6 +400,8 @@ struct tcp_sock { */ struct request_sock *fastopen_rsk; u32 *saved_syn; + + int rto_min; /* per sock tcp_rto_min */ }; enum tsq_enum { diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 2d123ea..373d3e0 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -169,6 +169,7 @@ struct netns_ipv4 { int sysctl_max_syn_backlog; int sysctl_tcp_fastopen; int sysctl_tcp_tw_timeout; + int sysctl_tcp_rto_min; const struct tcp_congestion_ops __rcu *tcp_congestion_control; struct tcp_fastopen_context __rcu *tcp_fastopen_ctx; spinlock_t tcp_fastopen_ctx_lock; diff --git a/include/net/snmp.h b/include/net/snmp.h index c9228ad..6fe87bb 100644 --- a/include/net/snmp.h +++ b/include/net/snmp.h @@ -153,6 +153,14 @@ struct linux_xfrm_mib { __this_cpu_add(ptr[basefield##OCTETS], addend); \ } while (0) +#define SNMP_UPD_STATS(mib, field, v) \ + do { \ + int i; \ + for_each_possible_cpu(i) \ + per_cpu(mib->mibs[field], i) = 0; \ + this_cpu_xchg(mib->mibs[field], v); \ + } while (0) + #if BITS_PER_LONG==32 diff --git a/include/net/tcp.h b/include/net/tcp.h index 7ec5fa2..f65ca74 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -313,6 +313,7 @@ static inline bool tcp_too_many_orphans(struct sock *sk, int shift) #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) +#define TCP_UPD_STATS(net, field, val) SNMP_UPD_STATS((net)->mib.tcp_statistics, field, val) void tcp_tasklet_init(void); @@ -687,7 +688,7 @@ static inline void tcp_fast_path_check(struct sock *sk) static inline u32 tcp_rto_min(struct sock *sk) { const struct dst_entry *dst = __sk_dst_get(sk); - u32 rto_min = TCP_RTO_MIN; + u32 rto_min = sock_net(sk)->ipv4.sysctl_tcp_rto_min; if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); @@ -1257,7 +1258,8 @@ static inline bool tcp_needs_internal_pacing(const struct sock *sk) */ static inline unsigned long tcp_probe0_base(const struct sock *sk) { - return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); + return max_t(unsigned long, inet_csk(sk)->icsk_rto, + sock_net(sk)->ipv4.sysctl_tcp_rto_min); } /* Variant of inet_csk_rto_backoff() used for zero window probes */ diff --git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h index c83686b..adc7679b 100644 --- a/include/uapi/linux/tcp.h +++ b/include/uapi/linux/tcp.h @@ -124,6 +124,7 @@ enum { #define TCP_FASTOPEN_NO_COOKIE 34 /* Enable TFO without a TFO cookie */ #define TCP_ZEROCOPY_RECEIVE 35 #define TCP_INQ 36 /* Notify bytes available to read as a cmsg on read */ +#define TCP_RTO_MINIMAL 99 /* TCP RTO minimal */ #define TCP_CM_INQ TCP_INQ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0f1ecba..48cc89a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3204,6 +3204,44 @@ static int do_proc_dointvec_ms_jiffies_conv(bool *negp, unsigned long *lvalp, return 0; } +struct do_proc_dointvec_ms_jiffies_minmax_conv_param { + int *min; + int *max; +}; + +static int do_proc_dointvec_ms_jiffies_minmax_conv(bool *negp, unsigned long *lvalp, + int *valp, + int write, void *data) +{ + struct do_proc_dointvec_ms_jiffies_minmax_conv_param *param = data; + + if (write) { + unsigned long jif = msecs_to_jiffies(*negp ? -*lvalp : *lvalp); + + if (jif > INT_MAX) + return -EINVAL; + + if ((param->min && *param->min > jif) || + (param->max && *param->max < jif)) + return -ERANGE; + + *valp = (int)jif; + } else { + int val = *valp; + unsigned long lval; + + if (val < 0) { + *negp = true; + lval = -(unsigned long)val; + } else { + *negp = false; + lval = (unsigned long)val; + } + *lvalp = jiffies_to_msecs(lval); + } + return 0; +} + /** * proc_dointvec_jiffies - read a vector of integers as seconds * @table: the sysctl table @@ -3271,6 +3309,18 @@ int proc_dointvec_ms_jiffies(struct ctl_table *table, int write, do_proc_dointvec_ms_jiffies_conv, NULL); } +int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + struct do_proc_dointvec_ms_jiffies_minmax_conv_param param = { + .min = (int *) table->extra1, + .max = (int *) table->extra2, + }; + return do_proc_dointvec(table, write, buffer, lenp, ppos, + do_proc_dointvec_ms_jiffies_minmax_conv, ¶m); +} + static int proc_do_cad_pid(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { @@ -3489,6 +3539,13 @@ int proc_dointvec_ms_jiffies(struct ctl_table *table, int write, return -ENOSYS; } +int proc_dointvec_ms_jiffies_minmax(struct ctl_table *table, int write, + void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + int proc_doulongvec_minmax(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { @@ -3520,3 +3577,4 @@ int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int write, EXPORT_SYMBOL(proc_dostring); EXPORT_SYMBOL(proc_doulongvec_minmax); EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); +EXPORT_SYMBOL(proc_dointvec_ms_jiffies_minmax); diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 08f3b21..fb45c24 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -54,6 +54,8 @@ static int one_day_secs = 24 * 3600; static int tcp_tw_timeout_min = 1 * HZ; static int tcp_tw_timeout_max = 600 * HZ; +static int tcp_rto_min_min = 1; /* one tick */ +static int tcp_rto_min_max = TCP_RTO_MAX; /* obsolete */ static int sysctl_tcp_low_latency __read_mostly; @@ -440,6 +442,24 @@ static int proc_fib_multipath_hash_policy(struct ctl_table *table, int write, } #endif +static int proc_tcp_rto_min(struct ctl_table *table, + int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + ret = proc_dointvec_ms_jiffies_minmax(table, write, buffer, lenp, ppos); + if (write && !ret) { + struct net *net = container_of(table->data, struct net, + ipv4.sysctl_tcp_rto_min); + int tcp_rto_min = init_net.ipv4.sysctl_tcp_rto_min; + + TCP_UPD_STATS(net, TCP_MIB_RTOMIN, tcp_rto_min * 1000 / HZ); + } + + return ret; +} + static struct ctl_table ipv4_table[] = { { .procname = "tcp_max_orphans", @@ -1234,6 +1254,15 @@ static int proc_fib_multipath_hash_policy(struct ctl_table *table, int write, .extra2 = &comp_sack_nr_max, }, { + .procname = "tcp_rto_min", + .data = &init_net.ipv4.sysctl_tcp_rto_min, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_tcp_rto_min, + .extra1 = &tcp_rto_min_min, + .extra2 = &tcp_rto_min_max + }, + { .procname = "udp_rmem_min", .data = &init_net.ipv4.sysctl_udp_rmem_min, .maxlen = sizeof(init_net.ipv4.sysctl_udp_rmem_min), diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 2dcbdd9..6b33811 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3062,6 +3062,12 @@ static int do_tcp_setsockopt(struct sock *sk, int level, else tp->recvmsg_inq = val; break; + case TCP_RTO_MINIMAL: + if (val > TCP_RTO_MAX || val < 0) + err = -EINVAL; + else + tp->rto_min = msecs_to_jiffies(val); + break; default: err = -ENOPROTOOPT; break; @@ -3528,6 +3534,9 @@ static int do_tcp_getsockopt(struct sock *sk, int level, case TCP_INQ: val = tp->recvmsg_inq; break; + case TCP_RTO_MINIMAL: + val = jiffies_to_msecs(tp->rto_min); + break; case TCP_SAVE_SYN: val = tp->save_syn; break; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 657b7db..5556d76 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2597,6 +2597,7 @@ static int __net_init tcp_sk_init(struct net *net) spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock); net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60; atomic_set(&net->ipv4.tfo_active_disable_times, 0); + net->ipv4.sysctl_tcp_rto_min = TCP_RTO_MIN; /* Reno is always built in */ if (!net_eq(net, &init_net) && diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 7ca9ceb..17a61ee 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -2444,7 +2444,10 @@ bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto) if (tp->srtt_us) { timeout = usecs_to_jiffies(tp->srtt_us >> 2); if (tp->packets_out == 1) - timeout += TCP_RTO_MIN; + if (tp->rto_min) + timeout += tp->rto_min; + else + timeout += sock_net(sk)->ipv4.sysctl_tcp_rto_min; else timeout += TCP_TIMEOUT_MIN; } else { diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index d1fa0a5..f9f553c 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -190,9 +190,15 @@ static bool retransmits_timed_out(struct sock *sk, unsigned int boundary, unsigned int timeout) { - const unsigned int rto_base = TCP_RTO_MIN; + struct tcp_sock *tp = tcp_sk(sk); + unsigned int rto_base; unsigned int linear_backoff_thresh, start_ts; + if (tp->rto_min) + rto_base = tp->rto_min; + else + rto_base = sock_net(sk)->ipv4.sysctl_tcp_rto_min; + if (!inet_csk(sk)->icsk_retransmits) return false; -- 1.8.3.1