[vox-tech] Very slow TCP transfer over loopback

[Mike Simons]

Hello All,

Got a mystery here... transferring data over the loopback interface
on a old RHAS 2.1 kernel (2.4.9-e.25enterprise) even a modern 2.6.1
kernel, is _very_ slow.  When the SO_RCVBUF is set low.

  As you can see from the tcpdump below after the session gets going,
only one data packet is exchanged every .2 seconds, the sending side
of the pipe seems to be waiting for something...

- Anyone have a place to look in the RFCs or such that would explain=20
  why it's waiting, for what?

    Thanks,
      Mike

ps: the RCVBUF can be changed upto 5460 bytes and same problem...

pps: if you move the RCVBUF setting to change the "accept_sock"
     instead of the "file", then the problem goes away regardless
     of what size.

tcpdump when RCVBUF is 1024
=3D=3D=3D
1078104350.943758 127.0.0.1.37612 > 127.0.0.1.2222: S 1206126103:1206126103=
(0) win 32767 <mss 16396,sackOK,timestamp 1050688520 0,nop,wscale 0> (DF)
1078104350.943788 127.0.0.1.2222 > 127.0.0.1.37612: S 1215430393:1215430393=
(0) ack 1206126104 win 32767 <mss 16396,sackOK,timestamp 1050688520 1050688=
520,nop,wscale 0> (DF)
1078104350.943801 127.0.0.1.37612 > 127.0.0.1.2222: . ack 1 win 32767 <nop,=
nop,timestamp 1050688520 1050688520> (DF)
1078104350.944058 127.0.0.1.37612 > 127.0.0.1.2222: P 1:8193(8192) ack 1 wi=
n 32767 <nop,nop,timestamp 1050688520 1050688520> (DF)
1078104350.944069 127.0.0.1.2222 > 127.0.0.1.37612: . ack 8193 win 24575 <n=
op,nop,timestamp 1050688520 1050688520> (DF)
1078104350.944143 127.0.0.1.37612 > 127.0.0.1.2222: P 8193:16385(8192) ack =
1 win 32767 <nop,nop,timestamp 1050688520 1050688520> (DF)
1078104350.944253 127.0.0.1.37612 > 127.0.0.1.2222: . 16385:32768(16383) ac=
k 1 win 32767 <nop,nop,timestamp 1050688521 1050688520> (DF)
1078104350.944292 127.0.0.1.2222 > 127.0.0.1.37612: . ack 32768 win 1536 <n=
op,nop,timestamp 1050688521 1050688520> (DF)
1078104350.944308 127.0.0.1.37612 > 127.0.0.1.2222: P 32768:32769(1) ack 1 =
win 32767 <nop,nop,timestamp 1050688521 1050688521> (DF)
1078104350.944321 127.0.0.1.2222 > 127.0.0.1.37612: . ack 32769 win 1536 <n=
op,nop,timestamp 1050688521 1050688521> (DF)
1078104351.145164 127.0.0.1.37612 > 127.0.0.1.2222: P 32769:34305(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050688722 1050688521> (DF)
1078104351.145219 127.0.0.1.2222 > 127.0.0.1.37612: . ack 34305 win 1536 <n=
op,nop,timestamp 1050688722 1050688722> (DF)
1078104351.346130 127.0.0.1.37612 > 127.0.0.1.2222: P 34305:35841(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050688923 1050688722> (DF)
1078104351.346185 127.0.0.1.2222 > 127.0.0.1.37612: . ack 35841 win 1536 <n=
op,nop,timestamp 1050688923 1050688923> (DF)
1078104351.547093 127.0.0.1.37612 > 127.0.0.1.2222: P 35841:37377(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050689124 1050688923> (DF)
1078104351.547147 127.0.0.1.2222 > 127.0.0.1.37612: . ack 37377 win 1536 <n=
op,nop,timestamp 1050689124 1050689124> (DF)
1078104351.748059 127.0.0.1.37612 > 127.0.0.1.2222: P 37377:38913(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050689325 1050689124> (DF)
1078104351.748111 127.0.0.1.2222 > 127.0.0.1.37612: . ack 38913 win 1536 <n=
op,nop,timestamp 1050689325 1050689325> (DF)
1078104351.949030 127.0.0.1.37612 > 127.0.0.1.2222: P 38913:40449(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050689526 1050689325> (DF)
1078104351.949091 127.0.0.1.2222 > 127.0.0.1.37612: . ack 40449 win 1536 <n=
op,nop,timestamp 1050689526 1050689526> (DF)
1078104352.150001 127.0.0.1.37612 > 127.0.0.1.2222: P 40449:41985(1536) ack=
 1 win 32767 <nop,nop,timestamp 1050689727 1050689526> (DF)
1078104352.150060 127.0.0.1.2222 > 127.0.0.1.37612: . ack 41985 win 1536 <n=
op,nop,timestamp 1050689727 1050689727> (DF)
=3D=3D=3D

server.c
=3D=3D=3D
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>

#define BEFORE_ACCEPT 0

int main(int argc, char *argv[])
{
  int accept_sock =3D socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
  int option =3D 1;
  int file;
  socklen_t accept_addr_len;
  struct sockaddr_in accept_addr;
  char buffer[1<<16];

  memset(&accept_addr, 0, sizeof(accept_addr));
  accept_addr.sin_family =3D AF_INET;
  accept_addr.sin_port =3D htons(2222);
  inet_aton("0.0.0.0", &accept_addr.sin_addr);

  setsockopt(accept_sock, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)=
);
  bind(accept_sock, (struct sockaddr *) &accept_addr, sizeof(accept_addr));

  listen(accept_sock, 128);

#ifdef BEFORE_ACCEPT
  if (!(argc =3D=3D 2 && sscanf(argv[1], "%d", &option)))
    option =3D 1024;

  setsockopt(accept_sock, SOL_SOCKET, SO_RCVBUF, &option, sizeof(option));
#endif

  accept_addr_len =3D sizeof(accept_addr);
  file =3D accept(accept_sock, (struct sockaddr *) &accept_addr, &accept_ad=
dr_len);

#ifndef BEFORE_ACCEPT
  if (!(argc =3D=3D 2 && sscanf(argv[1], "%d", &option)))
    option =3D 1024;

  setsockopt(accept_sock, SOL_SOCKET, SO_RCVBUF, &option, sizeof(option));
#endif
  while (read(file, &buffer, sizeof(buffer)));

  return 0;
}
=3D=3D=3D

How to run server:
=3D=3D=3D
msimons@fizban:~/www/tests/slow_loopback$ gcc -Wall -W s.c
msimons@fizban:~/www/tests/slow_loopback$ ./a.out

# to run with a given buffer size
msimons@fizban:~/www/tests/slow_loopback$ ./a.out 5460
=3D=3D=3D

How to run client:
=3D=3D=3D
msimons@fizban:~/www/tests/slow_loopback$ dd if=3D/dev/zero bs=3D1024 count=
=3D1024 | nc localhost 2222
=3D=3D=3D