Appendix 1: A TCP port forwarder

In this appendix all of the above themes are be re-used in the construction of a program that forwards a TCP port on the local host to a given port on a remote host. TCP traffic is shuttled to and fro. The constructed program will be appropriately called portfw, and requires three arguments:

Once started, the program will accept connections on the local TCP port. When activity is detected, then information is transmitted to and fro between the client that connects to the local port, and the remote port on the remote host.

E.g., after the invocation portfw 10000 thedailywtf.com 80 you can point your browser to http://localhost:10000/ and read The Daily WTF.

Port forwarding as such may not seem too useful, but the program is included here because it nicely illustrates all concepts of this series of lectures. Furthermore it can be used as a stepping stone for your own network related projects.

1.1: General design of the program

Before we dive into code, here's a few remarks about the general design of the program portfw.

Parallel processing: forking or threading?
The program must be able to handle lots of client connections in parallel. We're building for speed here: many clients must be able to connect to the port forwarder, and be served. Clients will need to be serviced in parallel. As described in chapter 2, separate requests could be either served from new forks, or from new threads. In this example we'll use threads. The program will only fork in order to become a daemon.

Messaging and program stages.
The program will use three messaging modes: for errors, for warnings, and for verbosity. The working of these will differ by program stage:

Not implemented features.
The program portfw could be enhanced with lots of extra features, which will be left to the reader. For example:

1.2: Technical description and sources

This section gives a technical overview of the program and shows the sources.

1.2.1: portfw.h: The central header

In this program all declarations and data definitions are collected in one header file, portfw.h. The inclusion of all necessary system headers also occurs here.

Noteworthy parts of the header are:

   1  /* portfw.h */
   2  
   3  #include <errno.h>
   4  #include <fcntl.h>
   5  #include <netdb.h>
   6  #include <pthread.h>
   7  #include <setjmp.h>
   8  #include <stdarg.h>
   9  #include <stdio.h>
  10  #include <stdlib.h>
  11  #include <string.h>
  12  #include <syslog.h>
  13  #include <unistd.h>
  14  #include <arpa/inet.h>
  15  #include <netinet/in.h>
  16  #include <sys/types.h>
  17  #include <sys/socket.h>
  18  
  19  /* Types. */
  20  typedef struct {
  21      int clientsockserversocksrcdst;
  22      int nreadnwrittentotwritten;
  23      unsigned totbytes;
  24      double starttime;
  25      fd_set readsetexceptset;
  26      char buf[1024];
  27  Clientinfo;
  28  
  29  /* Global variables. */
  30  #ifndef EXTERN
  31  #define EXTERN extern
  32  #endif
  33  EXTERN int is_daemon;
  34  EXTERN struct sockaddr_in remotesockaddr;
  35  
  36  /* Function prototypes */
  37  extern pid_t daemonize (void);
  38  extern void error (char const *fmt, ...);
  39  extern double gettime (void);
  40  extern void msg (char const *fmt, ...);
  41  extern void service_connect (Clientinfo *cl);
  42  extern void service_start (int locportchar const *remhostint remport);
  43  extern void service_run (int locsock);
  44  extern void *service_thread (void *);
  45  extern void service_stop (Clientinfo *clint is_errorchar const *fmt, ...);
  46  extern void warn (char const *fmt, ...);
  47  

1.2.2: The main function

The main() function, located in main.c, is straight forward. It is shown without further ado; all it does is verify that all commandline arguments are present, and then call service_start().

   1  /* main.c */
   2  
   3  #define EXTERN
   4  #include "portfw.h"
   5  
   6  int main (int argcchar **argv) {
   7      int localportremoteport;
   8      
   9      /* Verify arguments. */
  10      if (argc != 4)
  11          error ("Usage: portfw LOCALPORT REMOTEHOST REMOTEPORT\n"
  12                 "localhost:LOCALPORT will be forwarded to "
  13                 "REMOTEHOST:REMOTEPORT");
  14  
  15      /* Get ports. */
  16      if (sscanf (argv[1], "%d", &localport) < 1)
  17          error ("Bad local port specifier '%s'"argv[1]);
  18      if (sscanf (argv[3], "%d", &remoteport) < 1)
  19          error ("Bad remote port specifier '%s'"argv[3]);
  20  
  21      /* Start the service! */
  22      service_start (localportargv[2], remoteport);
  23  
  24      /* All done. */
  25      return (0);
  26  }
  27  

1.2.3: Messaging-related functions: error(), warn(), msg()

The following three listings show the messaging related functions. Depending on the above-discussed flag is_daemon, information is sent either to stderr or to syslog(). In the case of syslog() a "priority" flag states what the message is about. Function error() of course also terminates the program.

1.2.3.1: error()

   1  /* error.c */
   2  
   3  #include "portfw.h"
   4  
   5  void error (char const *fmt, ...) {
   6      va_list args;
   7  
   8      va_start (argsfmt);
   9      if (!is_daemon) {
  10          fprintf (stderr"ERROR: ");
  11          vfprintf (stderrfmtargs);
  12          fputc ('\n'stderr);
  13      } else
  14          vsyslog (LOG_ALERTfmtargs);
  15      
  16      exit (1);
  17  }
  18  

1.2.3.2: warn()

   1  /* warn.c */
   2  
   3  #include "portfw.h"
   4  
   5  void warn (char const *fmt, ...) {
   6      va_list args;
   7  
   8      if (!is_daemon) {
   9          va_start (argsfmt);
  10          fprintf (stderr"WARN: ");
  11          vfprintf (stderrfmtargs);
  12          fputc ('\n'stderr);
  13      } else
  14          vsyslog (LOG_WARNINGfmtargs);
  15  }
  16  

1.2.3.3: msg()

   1  /* msg.c */
   2  
   3  #include "portfw.h"
   4  
   5  void msg (char const *fmt, ...) {
   6      va_list args;
   7  
   8      va_start (argsfmt);
   9      if (!is_daemon) {
  10          fprintf (stderr"INFO: ");
  11          vfprintf (stderrfmtargs);
  12          fputc ('\n'stderr);
  13      } else
  14          vsyslog (LOG_NOTICEfmtargs);
  15  }
  16  

1.2.4: Service-related functions

1.2.4.1: service_start()

Function service_start() is called from main() and performs the following tasks:

   1  /* servicestart.c */
   2  
   3  #include "portfw.h"
   4  
   5  void service_start (int locportchar const *remhostint remport) {
   6      struct hostent *remhostent;
   7      int localsockval = 1;
   8      struct sockaddr_in localsockaddr;
   9      pid_t pid;
  10      
  11      msg ("Starting service localhost:%d -> %s:%d",
  12           locportremhostremport);
  13  
  14      /* Look up and name the remote host. */
  15      if (! (remhostent = gethostbyname (remhost)) )
  16          error ("Failed to resolve remote host '%s'"remhost);
  17      remotesockaddr.sin_family = AF_INET;
  18      remotesockaddr.sin_port = htons(remport);
  19      remotesockaddr.sin_addr = *(struct in_addr *)remhostent->h_addr;
  20  
  21      /* Set up the local listening port. */
  22      if ( (localsock = socket (PF_INETSOCK_STREAM, 0)) < 0)
  23          error ("Failed to create server socket: %s"strerror (errno));
  24      if (setsockopt (localsockSOL_SOCKETSO_REUSEADDR,
  25                      &valsizeof(val)))
  26          error ("Failed to set socket options: %s"strerror(errno));
  27      localsockaddr.sin_family = AF_INET;
  28      localsockaddr.sin_port = locport;
  29      localsockaddr.sin_addr.s_addr = htonl (INADDR_ANY);
  30  
  31      /* Bind the local socket */
  32      if (bind (localsock, (struct sockaddr *)&localsockaddr,
  33                sizeof(localsockaddr)) < 0)
  34          error ("Failed to bind local socket to port %d: %s",
  35                 locportstrerror(errno));
  36  
  37      /* Tell the kernel we're listening to it. */
  38      if (listen (localsock, 5) < 0)
  39          error ("Failed to listen to local socket: %s"strerror(errno));
  40  
  41      /* We're ready to start servicing. Daemonize! */
  42      if ( (pid = daemonize()) ) {
  43          /* Parent process */
  44          msg ("Open for business. Server started as PID %d."pid);
  45      } else {
  46          /* Daemon process. */
  47          service_run(localsock);
  48      }
  49  }
  50  

1.2.4.2: service_run()

The function service_run(), which is called from service_start(), is the event loop of the program. New network connections on the local socket are accepted, and when e new connection is ready, a thread context Clientinfo is prepared. After that a thread is started and executed via service_thread().

Note how service_run() doesn't exit once the event loop is running. Atmost, warn() is called to send an appropriate message to syslog(), after which the event loop is resumed.

   1  /* servicerun.c */
   2  
   3  #include "portfw.h"
   4  
   5  void service_run (int localsock) {
   6      int clientsocksize;
   7      struct sockaddr_in clientname;
   8      char *client_ip;
   9      fd_set readsetexceptset;
  10      Clientinfo *cl;
  11      pthread_t newthread;
  12  
  13      msg ("Running service, server socket: %d"localsock);
  14  
  15      while (1) {
  16          /* Wait for network activity */
  17          FD_ZERO (&readset);
  18          FD_SET (localsock, &readset);
  19          FD_ZERO (&exceptset);
  20          FD_SET (localsock, &exceptset);
  21  
  22          if (select (FD_SETSIZE, &readset, 0, &exceptset, 0) < 0)
  23              continue;
  24          if (FD_ISSET (localsock, &exceptset)) {
  25              warn ("Exception on network socket");
  26              continue;
  27          }
  28  
  29          /* We got action! Accept the network connection. */
  30          size = sizeof(clientname);
  31          if ( (clientsock = accept (localsock, (struct sockaddr *) &clientname,
  32                                     (socklen_t *) &size)) < 0) {
  33              warn ("Failed to accept network connection: %s"strerror(errno));
  34              continue;
  35          }
  36          client_ip = inet_ntoa (clientname.sin_addr);
  37          msg ("Accepted network connection from %s on fd %d",
  38               client_ipclientsock);
  39  
  40          /* Start serving the client's request, then continue to
  41           * listen to new requests.
  42           * We start up a new thread to serve the client, but mark it
  43           * ready for destruction right away. */
  44          if (! (cl = calloc (1, sizeof (Clientinfo)))) {
  45              warn ("Out of memory");
  46              close (clientsock);
  47          } else {
  48              cl->clientsock = clientsock;
  49              cl->starttime = gettime();
  50              if (pthread_create (&newthread, 0, service_threadcl)) {
  51                  warn ("Failed to start thread: %s"strerror(errno));
  52                  close (clientsock);
  53                  free (cl);
  54              }
  55          }
  56      }
  57  }
  58  

1.2.4.3: service_thread()

The actual servicing of a client connection occurs in service_thread(). It is called with an initialized context, stating a connected client socket, and a start time.

The servicing of a client connection means that first of all, a connection to the server must be established. Next, a loop is started what waits for one of the sockets to become readable (client-side or server-side). The readable socket is read, and anything that arrives there, is sent to the other socket.

A stop is detected when a network read returns that zero bytes were read.

   1  /* servicethread.c */
   2  
   3  #include "portfw.h"
   4  
   5  void *service_thread (void *ctx) {
   6      Clientinfo *cl = (Clientinfo *)ctx;
   7      
   8      /* Connect to the server. */
   9      if (! (cl->serversock = socket (PF_INETSOCK_STREAM, 0)) )
  10          service_stop (cl, 1, "Failed to create server-side socket: %s",
  11                         strerror(errno));
  12      service_connect (cl);
  13      
  14      /* Copy client to server and vice versa until one of them
  15       * terminates the connection. */
  16      while (1) {
  17          /* Wait until the client socket or the server socket become
  18           * readable. */
  19          FD_ZERO (&cl->readset);
  20          FD_SET (cl->clientsock, &cl->readset);
  21          FD_SET (cl->serversock, &cl->readset);
  22          FD_ZERO (&cl->exceptset);
  23          FD_SET (cl->clientsock, &cl->exceptset);
  24          FD_SET (cl->serversock, &cl->exceptset);
  25          if (select (FD_SETSIZE, &cl->readset, 0, &cl->exceptset, 0) < 0)
  26              continue;
  27  
  28          /* Check for exceptions on either side. */
  29          if (FD_ISSET (cl->clientsock, &cl->exceptset))
  30              service_stop (cl, 1, "Exception on client socket");
  31          if (FD_ISSET (cl->serversock, &cl->exceptset))
  32              service_stop (cl, 1, "Exception on server socket");
  33  
  34          /* Determine which socket is readable and copy from that one
  35           * to the other one. */
  36          if (FD_ISSET (cl->clientsock, &cl->readset)) {
  37              cl->src = cl->clientsock;
  38              cl->dst = cl->serversock;
  39              msg ("Activity on client socket %d, will copy to server socket %d",
  40                   cl->srccl->dst);
  41          } else if (FD_ISSET (cl->serversock, &cl->readset)) {
  42              cl->src = cl->serversock;
  43              cl->dst = cl->clientsock;
  44              msg ("Activity on server socket %d, will copy to client socket %d",
  45                   cl->srccl->dst);
  46          } else {
  47              warn ("Failed to determine src/dst network sockets");
  48              continue;
  49          }
  50  
  51          /* cl->src is the source fd to read, cl->dst is the dest to copy
  52           * to. Do the read, check errors */
  53          cl->nread = read (cl->srccl->buf, 1024);
  54          if (cl->nread < 0)
  55              service_stop (cl, 1, "Read failure on socket %d: %s",
  56                            cl->srcstrerror(errno));
  57          else if (cl->nread == 0)
  58              service_stop (cl, 0, "Succesful TCP stop after %.3f sec, "
  59                            "%u bytes transmitted",
  60                            (gettime() - cl->starttime) / 1000000, 
  61                            cl->totbytes);
  62          else {
  63              /* Read cl->nread bytes. Copy to cl->dst. */
  64              cl->totbytes += cl->nread;
  65              cl->totwritten = 0;
  66              while (cl->totwritten < cl->nread) {
  67                  cl->nwritten = write (cl->dstcl->buf + cl->totwritten,
  68                                        cl->nread - cl->totwritten);
  69                  if (cl->nwritten < 1)
  70                      service_stop (cl, 1, "Write failure on socket %d: %s",
  71                                    cl->dststrerror(errno));
  72                  cl->totwritten += cl->nwritten;
  73              }   
  74          }
  75      }
  76  }
  77  

1.2.4.4: service_connect()

The helper function service_connect() is called from service_thread(). It is responsible for establishing a connection to a server back end. The shown code demonstrates how to check for timeouts during connect() in threaded programs.

   1  #include "portfw.h"
   2  
   3  /* Connect cl->serversock to remotesockaddr */
   4  void service_connect (Clientinfo *cl) {
   5      int flagsnfd;
   6      pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
   7      fd_set writesetexceptset;
   8      struct timeval tv;
   9  
  10      /* Put socket in nonblocking mode */
  11      if ( (flags = fcntl (cl->serversockF_GETFL, 0)) == -1 )
  12          service_stop (cl, 1, "Failed to get socket flags: %s",
  13                        strerror(errno));
  14      if (fcntl (cl->serversockF_SETFLflags | O_NONBLOCK) == -1)
  15          service_stop (cl, 1, "Failed to make socket nonblocking: %s",
  16                        strerror(errno));
  17  
  18      /* Connect. Errno EINPROGRESS is acceptable. Use mutex locks, as
  19       * errno is a static variable, multiple threads might change it
  20       * at the same time. */
  21      pthread_mutex_lock (&mutex);
  22      if (connect (cl->serversock, (struct sockaddr *)&remotesockaddr,
  23                   sizeof(remotesockaddr)) &&
  24          errno != EINPROGRESS)
  25          service_stop (cl, 1, "Failed to connect to server: %s",
  26                        strerror(errno));
  27      pthread_mutex_unlock (&mutex);
  28      
  29      /* Socket can go in blocking mode again. */
  30      if (fcntl (cl->serversockF_SETFLflags) == -1)
  31          service_stop (cl, 1, "Failed to make socket blocking: %s",
  32                        strerror(errno));
  33  
  34      /* Assuming a 5 sec delay, wait for the socket to become writable. */
  35      FD_ZERO (&writeset);
  36      FD_SET (cl->clientsock, &writeset);
  37      FD_ZERO (&exceptset);
  38      FD_SET (cl->clientsock, &exceptset);
  39      tv.tv_sec = 6;
  40      tv.tv_usec = 0;
  41      if ( (nfd = select (FD_SETSIZE, 0, &writeset, &exceptset, &tv)) < 0 )
  42          service_stop (cl, 1, "Select failed after connecting: %s",
  43                        strerror(errno));
  44      else if (!nfd)
  45          service_stop (cl, 1, "Connection to server timed out");
  46  }
  47      
  48  

1.2.4.5: service_stop()

All thread-related exits are combined into function service_stop(). Its first argument is the thread context; actually a pointer to a Clientinfo structure that needs to be freed. Also, service_stop() closes the two network sockets that belong to the thread.

   1  /* servicestop.c */
   2  
   3  #include "portfw.h"
   4  
   5  void service_stop (Clientinfo *clint is_errorchar const *fmt, ...) {
   6      va_list args;
   7      char buf[1024];
   8  
   9      va_start (argsfmt);
  10      vsprintf (buffmtargs);
  11      if (is_error)
  12          warn (buf);
  13      else
  14          msg (buf);
  15      close (cl->clientsock);
  16      if (cl->serversock)
  17          close (cl->serversock);
  18      free (cl);
  19      pthread_exit (0);
  20  }
  21  

1.2.5: Other support functions

The function daemonize() was discussed before. This version performs the same tasks, but also raises the flag is_daemon.

   1  /* daemonize.c */
   2  
   3  #include "portfw.h"
   4  
   5  pid_t daemonize () {
   6      pid_t pid;
   7  
   8      if ( (pid = fork()) < 0 ) {
   9          error ("Fork failure: %s"strerror(errno));
  10      } else if (pid > 0) {
  11          /* Parent branch */
  12          return (pid);
  13      }
  14      
  15      /* We now must be the child branch. Go to the root dir. */
  16      if (chdir ("/"))
  17          error ("Cannot chdir to /: %s"strerror(errno));
  18  
  19      /* Become session leader of a new process group. */
  20      if (setsid() == -1)
  21          error ("Failed to create process group. Already crated?");
  22  
  23      /* Close FD's 0,1,2 and reopen them on /dev/null. Note that we
  24       * cannot report on failures of the open() below -- stderr is
  25       * very likely already gone. */
  26      close (0);
  27      close (1);
  28      close (2);
  29      open ("/dev/null"O_RDONLY);
  30      open ("/dev/null"O_WRONLY);
  31      open ("/dev/null"O_WRONLY);
  32  
  33      /* We're a daemon now! */
  34      is_daemon++;
  35      return (0);
  36  }
  37  

The function gettime() returns the time since 'start of epoch' January 1st 1970 GMT. However, the time is returned as the number of microseconds. Calling gettime() repeatedly, and subtracting the returned values, provides as high resolution timer. This function is called from two places:

   1  /* gettime.c */
   2  
   3  #include "portfw.h"
   4  
   5  double gettime () {
   6      struct timeval tv;
   7  
   8      gettimeofday (&tv, 0);
   9      return (tv.tv_sec * 1000000 + tv.tv_usec);
  10  }
  11  

1.2.6: The Makefile

For completeness' sake here is also a Makefile, suitable for GNU Make.

# Central Makefile for portfw
# ---------------------------

SRC = $(wildcard *.c)
OBJ = $(patsubst %.c, %.o, $(SRC))
BIN = portfw

%.o: %.c
	$(CC) -g -O2 -Wall -c -o $@ $<

$(BIN): $(OBJ)
	$(CC) -o $(BIN) $(OBJ)
clean:
	rm -f $(OBJ)
main.o: main.c portfw.h Makefile