A Step-by-Step guide to making your own Linux kernel module that
intercepts custom system calls.  By Craig Kelley -- Spring 2001

--------
Step One : Make sure your kernel is ready
--------
Download a kernel from kernel.org and unpack it in /usr/src.  This
will create a linux/ directory, so if you already have one there,
please remove the old one.

Verify that the system headers now point to the correct location in
your kernel tree; the following symbolic links need to be exactly as
shown: 

  /usr/include/linux -> /usr/src/linux/include/linux
  /usr/include/asm -> /usr/src/linux/include/asm

(Otherwise, your header files will not be correct and there will be no
way to reference your new system call from userland!)

--------
Step Two : Install a new system call handler in your kernel
--------
System calls are named in two places inside the kernel:

  /usr/src/linux/include/asm/unistd.h
  /usr/src/linux/arch/i386/kernel/entry.S

In unistd.h, you will see a table like this:

#define __NR_exit                 1
#define __NR_fork                 2
#define __NR_read                 3
#define __NR_write                4
#define __NR_open                 5

  [snip]

Now, go to the end of that table and insert a custom system call (I am
calling mine 'cs584'):

  [snip]

#define __NR_madvise1           219     /* delete when C lib stub is removed */
#define __NR_getdents64         220
#define __NR_fcntl64            221
#define __NR_gettid             222
#define __NR_cs584              223     /* Craig's custom syscall */

  [snip]

Next you need to edit entry.S, you will see a table like this:

  [snip]

ENTRY(sys_call_table)
        .long SYMBOL_NAME(sys_ni_syscall)       /* 0  -  old "setup()" system call*/
        .long SYMBOL_NAME(sys_exit)
        .long SYMBOL_NAME(sys_fork)
        .long SYMBOL_NAME(sys_read)
        .long SYMBOL_NAME(sys_write)

  [snip]

Add another new entry at the bottom like this:

  [snip]

        .long SYMBOL_NAME(sys_fcntl64)
        .long SYMBOL_NAME(sys_gettid)
        .long SYMBOL_NAME(sys_ni_syscall)       /* reserved for TUX */
        .long SYMBOL_NAME(sys_cs584)         /* Craig's custom syscall */

  [snip]

And then increment this line:

        .rept NR_syscalls-223

like so:

        .rept NR_syscalls-224

(this tells the kernel how much space to reserve for system calls)

----------
Step Three:  Write a stub system call that does nothing
----------
Now we need to actually write a system call stub so that the kernel
compiles fine, and if anyone happens to call this system call it
should do nothing at all.  The best place to put new system calls in
the kernel is in /usr/src/linux/kernel.  I created a new file called
(you guessed it) cs584.c

  [snip]

/*
 * a wrapper for a custom system call
 */

#include <linux/mm.h>
#include <linux/utsname.h>
#include <linux/mman.h>
#include <linux/smp_lock.h>
#include <linux/notifier.h>
#include <linux/reboot.h>
#include <linux/prctl.h>
#include <linux/init.h>
#include <linux/highuid.h>

#include <asm/uaccess.h>

extern asmlinkage int sys_cs584(void *);

/*
 * a stub to be filled in by a module
 */
 
asmlinkage int sys_cs584(void *data)
{
  return 0;
}

  [snip]

All this does is (1) creates a new system call and (2) makes a stub
that returns zero, which means that everything is okay.  Now, we need
to tell the kernel build process to include this file when compiling
the kernel; in the file /usr/src/linux/kernel/Makefile there is a
macro named 'obj-y'.  Simply append your object file to the end of
that list:

  [snip]

obj-y     = sched.o dma.o fork.o exec_domain.o panic.o printk.o \
            module.o exit.o itimer.o info.o time.o softirq.o resource.o \
            sysctl.o acct.o capability.o ptrace.o timer.o user.o \
            signal.o sys.o kmod.o context.o cs584.o

  [snip]

Now save your kernel configuration and clean out all the dependencies
and rebuild your kernel (if you do not have a kernel configuration,
then you will need to create one using `make menuconfig` or some
such):

cd /usr/src/linux
mv -f .config ..
make distclean
mv -f ../.config .
make menuconfig
  [verify everything / setup the kernel / exit and save changes]
make dep && make bzImage
make modules && make modules_install

Then boot up with the new kernel (in /usr/src/linux/arch/i386/boot/bzImage)
and you should now have a system call installed!

---------
Step Four:  Test out your kernel by writing a module:
---------

Here is a simple kernel module which will replace our dummy system
call:

  [snip]

#include <linux/module.h>
#include <linux/kernel.h>
#include <asm/unistd.h>
#include <sys/syscall.h>

extern void* sys_call_table[];       /*sys_call_table is exported, so we
                                     can access it*/               

int (*orig_cs584)(void *); /*the original systemcall*/


int new_cs584(void *data)
{
  
  printk ("You called the new cs584 system call!\n");

  return 0;
}

int init_module(void)                /*module setup*/
{

  printk("Installing new sytem call handler for cs584\n");

  orig_cs584=sys_call_table[__NR_cs584];
  sys_call_table[__NR_cs584]=new_cs584;

  printk("New sytem call handler for cs584 installed\n");
  return 0;
}

void cleanup_module(void)            /*module shutdown*/
{

  printk("Installing original sytem call handler for cs584\n");

  sys_call_table[__NR_cs584]=orig_cs584; /*set cs584 syscall to the origal
                                       one*/
  printk("Original system call handler for cs584 installed\n");
}

  [snip]

Notice that the init_module saves the current handler (our dummy
handler inside the kernel) and then replaces it with a new one defined
in this code.  It also writes to the system log, so you should see the
output in your logfile when you run `insmod ./my_module.o`:

Apr 10 15:54:49 desconocido kernel: Installing new sytem call handler for cs584
Apr 10 15:54:49 desconocido kernel: New sytem call handler for cs584 installed


Then, we can run this sample user-land program to test it out:

  [snip]

#include <stdio.h>
#include <linux/unistd.h>

_syscall1(int, cs584, void*, data);

int main (int argc, char *argv[]) {

  void *data;

  printf ("Program started.\n");
  printf ("Making system call:");

  cs584(data);

  printf ("\nDone making system call.\n");

  return (0);

}

  [snip]

And the kernel log file should say:

Apr 10 15:58:22 desconocido kernel: You called the new cs584 system call!

Now remove the module with rmmod `rmmod my_module`, and the log file
will report:

Apr 10 15:58:44 desconocido kernel: Installing original sytem call handler for cs584
Apr 10 15:58:44 desconocido kernel: Original system call handler for cs584 installed

--------

That's it; from here you can replace the my_module code with whatever
you want to.  It accepts a void* argument, so you can pass in/out
whatever data you want from userland to the kernel.

All the code to do this (and Makefiles) is included with this file.