Installing ISPCP On Ubuntu Lucid x86_64 And Implementing Chroot Per Virtual Host In Apache2
Install server
Install a basic Ubuntu Lucid x86_64 server by following the first two pages of The Perfect Server - Ubuntu Lucid Lynx (Ubuntu 10.04) [ISPConfig 2] howto.
Install ISPCP
Download the ISPCP sources from the official download page. This guide is based on 1.0.6. Extract it, and change into the extracted directory.
tar xf iscp-omega-1.0.6.tar.gz
cd ispcp-omega-1.0.6
cp docs/Ubuntu/ubuntu-packages-karmic docs/Ubuntu/ubuntu-packages-lucid
Edit docs/Ubuntu/ubuntu-packages-lucid, and replace these entries:
libmd5-perl libmysqlclient15off
with these:
libdigestmd5-perl libmysqlclient16
Follow the installation steps detailed in docs/Ubuntu/INSTALL. When running ispcp-setup, make sure to select FastCGI. Your default ISPCP system should work at this point, and ps auxf output should look very similar to this:
root 25761 0.0 1.7 138304 36360 ? Ss 10:56 0:00 /usr/sbin/apache2 -k start root 25762 0.0 0.0 4096 580 ? S 10:56 0:00 \_ /bin/sh -c /var/www/ispcp/engine/ispcp-apache-logger -e root 25764 0.0 0.7 62624 15416 ? S 10:56 0:00 | \_ /usr/bin/perl /var/www/ispcp/engine/ispcp-apache-logger -e root 25763 0.0 0.0 4096 576 ? S 10:56 0:00 \_ /bin/sh -c /var/www/ispcp/engine/ispcp-apache-logger root 25765 0.0 0.7 62624 15416 ? S 10:56 0:00 | \_ /usr/bin/perl /var/www/ispcp/engine/ispcp-apache-logger www-data 25766 0.0 1.6 138004 34416 ? S 10:56 0:00 \_ /usr/sbin/apache2 -k start www-data 25767 0.0 1.6 138004 34372 ? S 10:56 0:00 \_ /usr/sbin/fcgi-pm -k start vu2000 25768 0.0 0.4 147120 8936 ? Ss 10:56 0:00 | \_ /usr/bin/php5-cgi vu2000 25826 0.0 0.7 155448 15156 ? S 10:56 0:00 | \_ /usr/bin/php5-cgi vu2000 25827 0.0 0.1 147120 3788 ? S 10:56 0:00 | \_ /usr/bin/php5-cgi www-data 25769 0.0 1.8 427808 37468 ? Sl 10:56 0:00 \_ /usr/sbin/apache2 -k start www-data 25778 0.0 1.7 427688 36796 ? Sl 10:56 0:00 \_ /usr/sbin/apache2 -k start
Compile Custom Suexec Wrapper
Most of this section is based on the excellent post at metaclarity. Download the Apache HTTPd sources from the official download page. This guide is based on 2.2.16.
tar xf httpd-2.2.16.tar.bz2
cd httpd-2.2.16/support
Apply the patches.
--- Makefile.in.old 2009-04-21 09:59:33.000000000 -0400 +++ Makefile.in 2009-04-21 10:04:11.000000000 -0400 @@ -59,9 +59,13 @@ $(LINK) $(checkgid_LTFLAGS) $(checkgid_OBJECTS) $(PROGRAM_LDADD) suexec_OBJECTS = suexec.lo -suexec: $(suexec_OBJECTS) +suexec: $(suexec_OBJECTS) suexec-phpfcgi $(LINK) $(suexec_OBJECTS) +suexec-phpfcgi_OBJECTS = suexec-phpfcgi.lo +suexec-phpfcgi: $(suexec-phpfcgi_OBJECTS) + $(LINK) $(suexec-phpfcgi_OBJECTS) + htcacheclean_OBJECTS = htcacheclean.lo htcacheclean: $(htcacheclean_OBJECTS) $(LINK) $(htcacheclean_LTFLAGS) $(htcacheclean_OBJECTS) $(PROGRAM_LDADD)
--- suexec.c 2010-07-12 11:26:29.000000000 +0200 +++ suexec-phpfcgi.c 2010-09-14 10:07:36.581742585 +0200 @@ -259,6 +259,7 @@ char *cmd; /* command to be executed */ char cwd[AP_MAXPATH]; /* current working directory */ char dwd[AP_MAXPATH]; /* docroot working directory */ + char nwd[AP_MAXPATH]; /* after-chroot working dir */ struct passwd *pw; /* password entry holder */ struct group *gr; /* group entry holder */ struct stat dir_info; /* directory info holder */ @@ -456,7 +457,6 @@ log_err("cannot run as forbidden uid (%d/%s)\n", uid, cmd); exit(107); } - /* * Error out if attempt is made to execute as root group * or as a GID less than AP_GID_MIN. Tsk tsk. @@ -466,6 +466,42 @@ exit(108); } + + int striplen = strlen (target_homedir); + + char* tlen = strchr(target_homedir, '/'); + char* hlen = strchr(tlen+1, '/'); + char* ulen = strchr(hlen+1, '/'); + char* chroot_dir = strndup(target_homedir, striplen); + char* pt = getenv("PATH_TRANSLATED"); + if (pt != 0) { + setenv("PATH_TRANSLATED", pt + (ulen - target_homedir), 1); + } + + setenv("DOCUMENT_ROOT", "/", 1); + + if (getcwd(nwd, AP_MAXPATH) == NULL) { + log_err("cannot get current working directory (prechroot)\n"); + exit(111); + } + + char* trunc_nwd = strndup("/fcgi", 5); + + if (chdir(chroot_dir)) { + log_err("crit: can't chdir to chroot dir (%s)",chroot_dir); + exit(121); + } + + if (chroot(chroot_dir) != 0) { + log_err("emerg: failed to chroot (%s, %s)\n", chroot_dir, cmd); + exit(122); + } + + if (chdir (trunc_nwd)) { + log_err("crit: can't chdir to %s inside chroot %s\n", trunc_nwd, chroot_dir); + exit(42); + } + /* * Change UID/GID here so that the following tests work over NFS. * @@ -498,22 +534,11 @@ exit(111); } - if (userdir) { - if (((chdir(target_homedir)) != 0) || - ((chdir(AP_USERDIR_SUFFIX)) != 0) || - ((getcwd(dwd, AP_MAXPATH)) == NULL) || - ((chdir(cwd)) != 0)) { - log_err("cannot get docroot information (%s)\n", target_homedir); - exit(112); - } - } - else { - if (((chdir(AP_DOC_ROOT)) != 0) || - ((getcwd(dwd, AP_MAXPATH)) == NULL) || - ((chdir(cwd)) != 0)) { - log_err("cannot get docroot information (%s)\n", AP_DOC_ROOT); - exit(113); - } + if (((chdir(getenv("DOCUMENT_ROOT"))) != 0) || + ((getcwd(dwd, AP_MAXPATH)) == NULL) || + ((chdir(cwd)) != 0)) { + log_err("cannot get docroot information (%s)\n", AP_DOC_ROOT); + exit(113); } if ((strncmp(cwd, dwd, strlen(dwd))) != 0) { @@ -565,7 +590,7 @@ * Error out if the target name/group is different from * the name/group of the cwd or the program. */ - if ((uid != dir_info.st_uid) || +/* if ((uid != dir_info.st_uid) || (gid != dir_info.st_gid) || (uid != prg_info.st_uid) || (gid != prg_info.st_gid)) { @@ -575,7 +600,7 @@ dir_info.st_uid, dir_info.st_gid, prg_info.st_uid, prg_info.st_gid); exit(120); - } + }*/ /* * Error out if the program is not executable for the user. * Otherwise, she won't find any error in the logs except for
patch -p0 < ../../makefile.diff
patch -p0 -o suexec-phpfcgi.c < ../../suexec-phpfcgi.diff
Edit suexec.h:
AP_HTTPD_USER www-data AP_UID_MIN 2000 AP_GID_MIN 2000 AP_LOG_EXEC "/tmp/suexec.log" AP_DOC_ROOT "/"
cd ..
configure --prefix=/usr/lib/apache2 --enable-suexec --enable-mods-shared=most --enable-so --with-mpm=worker --with-included-apr
cd support
make suexec
Install the new suexec wrapper.
cp suexec-phpfcgi /usr/lib/apache2
chown .www-data /usr/lib/apache2/suexec-phpfcgi
chmod 4751 /usr/lib/apache2/suexec-phpfcgi
Update /etc/apache2/modules-enables/fastcgi_ispcp.conf to use the new wrapper:
FastCgiWrapper /usr/lib/apache2/suexec-phpfcgi
Create Chroot For Virtual Hosts
The following is a sufficient chroot. I put mine in /usr/local/skel. You may want to add more/other timezones.
. \__ bin -> chroot/bin \__ chroot | \__ bin | | \__ dash | | \__ sh -> dash | \__ dev | | \__ null | | \__ urandom | | \__ zero | \__ etc | | \__ group | | \__ host.conf | | \__ hosts | | \__ locale.alias | | \__ localtime | | \__ nsswitch.conf | | \__ passwd | | \__ php5 | | | \__ cgi | | | | \__ conf.d -> ../conf.d | | | | \__ php.ini | | | \__ conf.d | | | | \__ adodb.ini | | | | \__ gd.ini | | | | \__ mcrypt.ini | | | | \__ mysqli.ini | | | | \__ mysql.ini | | | | \__ pdo.ini | | | | \__ pdo_mysql.ini | | | \__ php.ini | | \__ protocols | | \__ resolv.conf | | \__ services | \__ lib | | \__ ld-2.11.1.so | | \__ ld-linux-x86-64.so.2 -> ld-2.11.1.so | | \__ libbz2.so.1 -> libbz2.so.1.0.4 | | \__ libbz2.so.1.0 -> libbz2.so.1.0.4 | | \__ libbz2.so.1.0.4 | | \__ libc-2.11.1.so | | \__ libcom_err.so.2 -> libcom_err.so.2.1 | | \__ libcom_err.so.2.1 | | \__ libcrypt-2.11.1.so | | \__ libcrypto.so.0.9.8 | | \__ libcrypt.so.1 -> libcrypt-2.11.1.so | | \__ libc.so.6 -> libc-2.11.1.so | | \__ libdl-2.11.1.so | | \__ libdl.so.2 -> libdl-2.11.1.so | | \__ libexpat.so.1 -> libexpat.so.1.5.2 | | \__ libexpat.so.1.5.2 | | \__ libgcc_s.so.1 | | \__ libkeyutils-1.2.so | | \__ libkeyutils.so.1 -> libkeyutils-1.2.so | | \__ libm-2.11.1.so | | \__ libm.so.6 -> libm-2.11.1.so | | \__ libncurses.so.5 -> libncurses.so.5.7 | | \__ libncurses.so.5.7 | | \__ libnsl-2.11.1.so | | \__ libnsl.so.1 -> libnsl-2.11.1.so | | \__ libnss_dns-2.11.1.so | | \__ libnss_dns.so.2 -> libnss_dns-2.11.1.so | | \__ libnss_files-2.11.1.so | | \__ libnss_files.so.2 -> libnss_files-2.11.1.so | | \__ libnss_nis-2.11.1.so | | \__ libnss_nis.so.2 -> libnss_nis-2.11.1.so | | \__ libpam.so.0 -> libpam.so.0.82.2 | | \__ libpam.so.0.82.2 | | \__ libpcre.so.3 -> libpcre.so.3.12.1 | | \__ libpcre.so.3.12.1 | | \__ libpng12.so.0 -> libpng12.so.0.42.0 | | \__ libpng12.so.0.42.0 | | \__ libpthread-2.11.1.so | | \__ libpthread.so.0 -> libpthread-2.11.1.so | | \__ libresolv-2.11.1.so | | \__ libresolv.so.2 -> libresolv-2.11.1.so | | \__ librt-2.11.1.so | | \__ librt.so.1 -> librt-2.11.1.so | | \__ libselinux.so.1 | | \__ libssl.so.0.9.8 | | \__ libz.so.1 -> libz.so.1.2.3.3 | | \__ libz.so.1.2.3.3 | \__ lib64 -> lib | \__ usr | \__ bin | | \__ php5-cgi | \__ lib | | \__ apache2 | | | \__ logs -> /var/log/apache2 | | \__ gconv | | | \__ gconv-modules | | | \__ gconv-modules.cache | | \__ libdb-4.8.so | | \__ libfontconfig.so.1 -> libfontconfig.so.1.4.4 | | \__ libfontconfig.so.1.4.4 | | \__ libfreetype.so.6 -> libfreetype.so.6.3.22 | | \__ libfreetype.so.6.3.22 | | \__ libgd.so.2 -> libgd.so.2.0.0 | | \__ libgd.so.2.0.0 | | \__ libgssapi_krb5.so.2 -> libgssapi_krb5.so.2.2 | | \__ libgssapi_krb5.so.2.2 | | \__ libjpeg.so.62 -> libjpeg.so.62.0.0 | | \__ libjpeg.so.62.0.0 | | \__ libk5crypto.so.3 -> libk5crypto.so.3.1 | | \__ libk5crypto.so.3.1 | | \__ libkrb5.so.3 -> libkrb5.so.3.3 | | \__ libkrb5.so.3.3 | | \__ libkrb5support.so.0 -> libkrb5support.so.0.1 | | \__ libkrb5support.so.0.1 | | \__ libmcrypt.so.4 -> libmcrypt.so.4.4.8 | | \__ libmcrypt.so.4.4.8 | | \__ libmysqlclient_r.so.16 -> libmysqlclient_r.so.16.0.0 | | \__ libmysqlclient_r.so.16.0.0 | | \__ libt1.so.5 -> libt1.so.5.1.2 | | \__ libt1.so.5.1.2 | | \__ libX11.so.6 -> libX11.so.6.3.0 | | \__ libX11.so.6.3.0 | | \__ libXau.so.6 -> libXau.so.6.0.0 | | \__ libXau.so.6.0.0 | | \__ libxcb.so.1 -> libxcb.so.1.1.0 | | \__ libxcb.so.1.1.0 | | \__ libXdmcp.so.6 -> libXdmcp.so.6.0.0 | | \__ libXdmcp.so.6.0.0 | | \__ libxml2.so.2 -> libxml2.so.2.7.6 | | \__ libxml2.so.2.7.6 | | \__ libXpm.so.4 -> libXpm.so.4.11.0 | | \__ libXpm.so.4.11.0 | | \__ locale | | | \__ en_ZA.utf8 | | | \__ LC_ADDRESS | | | \__ LC_COLLATE | | | \__ LC_CTYPE | | | \__ LC_IDENTIFICATION | | | \__ LC_MEASUREMENT | | | \__ LC_MESSAGES | | | | \__ SYS_LC_MESSAGES | | | \__ LC_MONETARY | | | \__ LC_NAME | | | \__ LC_NUMERIC | | | \__ LC_PAPER | | | \__ LC_TELEPHONE | | | \__ LC_TIME | | \__ php5 | | | \__ 20090626 | | | \__ adodb.so | | | \__ gd.so | | | \__ mcrypt.so | | | \__ mysqli.so | | | \__ mysql.so | | | \__ pdo_mysql.so | | | \__ pdo.so | | \__ sendmail -> ../sbin/mini_sendmail | \__ lib64 -> lib | \__ sbin | | \__ mini_sendmail | | \__ sendmail -> mini_sendmail | \__ share | \__ locale | | \__ locale.alias -> /etc/locale.alias | \__ zoneinfo | \__ Africa | \__ Johannesburg \__ dev -> chroot/dev \__ etc -> chroot/etc \__ fcgi | \__ php5-fcgi-starter \__ lib -> chroot/lib \__ lib64 -> chroot/lib64 \__ logs \__ tmp \__ usr -> chroot/usr \__ var \__ lib | \__ apache2 | \__ php5 \__ log | \__ apache2 -> /logs \__ www \__ virtual
Complete Chroot
For every virtual host created, the chroot as defined above should be copied into the home directory of the virtual host. For example, for virtual host example.org which runs as user vu2009, the following applies:
cp -a /usr/local/skel/* ~vu2009/
Additionally, the following is required:
mkdir -p ~vu2009/var/www/virtual/example.org
cd ~vu2009/var/www/virtual/example.org && ln -s /htdocs htdocs
Also create a link for every subdomain. If you have subdomains one.example.org and two.example.org, also do:
cd ~vu2009/var/www/virtual/example.org && ln -s /one one
cd ~vu2009/var/www/virtual/example.org && ln -s /two two
Note that these symlinks appear broken. They do, however, work when you take into consideration that ~vu2009 is the root of the chroot'ed filesystem.
Update ispcp.conf
Make sure database is using TCP (127.0.0.1), not sockets (localhost). The chroot'ed vhost will not be able to see the MySQL socket which resides in /var/run/mysqld/mysqld.sock.
DATABASE_HOST = 127.0.0.1
Update Master Virtual Host
Need a bit more symlink magic.
cd /var/www/fcgi/master && ln -s / fcgi
cp -a /var/www/fcgi/master /var/www/ispcp/gui/var/www/fgci/master
Update /etc/passwd, change vu2000 (master user) home directory to /var/www/ispcp/gui. The control panel needs to be able to read the main config. This file never changes, so a direct copy is fine.
cp /etc/ispcp/ispcp.conf /var/www/ispcp/gui/etc/ispcp/
cd /var/www/ispcp/gui/var/www/ispcp/ && ln -s / gui
Chroot Apache2
mkdir /var/www-jail
Update /etc/apache2/httpd.conf:
ChRootDir /var/www-jail
mkdir -p /var/www-jail/var/www
mount -o bind /var/www /var/www-jail/var/www
Once chroot'ed, the FastCGI sockets need to be available to the vhosts.
mount -o bind /var/lib/apache2/fastcgi /var/www-jail/var/lib/apache2/fastcgi
Make these mounts permanent by adding to /etc/fstab:
/var/www /var/www-jail/var/www none bind /var/lib/apache2/fastcgi /var/www-jail/var/lib/apache2/fastcgi none bind
Sendmail Mini
Postfix has a LOT of dependencies, so it's preferable to not include it in the vhosts' chroots. Instead, download and compile mini_sendmail from here. It wouldn't work statically linked, but it requires no more libs than we already have in the chroots, so it's fine. Edit the Makefile:
CFLAGS = -g #LDFLAGS = -s -static LDFLAG = -g
We don't give nearly enough information for the wrapper to be happy, so we change it somewhat:
--- mini_sendmail-1.3.6/mini_sendmail.c 2005-06-29 19:37:15.000000000 +0200 +++ mini_sendmail-1.3.6-mine/mini_sendmail.c 2010-10-01 15:12:40.947187053 +0200 @@ -30,7 +30,7 @@ ** don't want the features you can undef the symbols; some of them mean ** a significant savings in executable size. */ -#define DO_RECEIVED /* whether to add a "Received:" header */ +#undef DO_RECEIVED /* whether to add a "Received:" header */ #define DO_GETPWUID /* whether to try a getpwuid() if getlogin() fails */ #define DO_MINUS_SP /* whether to implement the -s and -p flags */ #define DO_DNS /* whether to do a name lookup on -s, or just IP# */ @@ -122,6 +122,8 @@ { if ( strncmp( argv[argn], "-f", 2 ) == 0 && argv[argn][2] != '\0' ) fake_from = &(argv[argn][2]); + else if ( strncmp( argv[argn], "-f", 2) == 0) + fake_from = &(argv[++argn][0]); else if ( strcmp( argv[argn], "-t" ) == 0 ) parse_message = 1; #ifdef DO_MINUS_SP @@ -145,33 +147,32 @@ ++argn; } - username = getlogin(); - if ( username == (char*) 0 ) - { + if ( fake_from == (char*) 0 ) { + username = getlogin(); + if ( username == (char*) 0 ) { #ifdef DO_GETPWUID - struct passwd* pw = getpwuid( getuid() ); - if ( pw == (struct passwd*) 0 ) - { - (void) fprintf( stderr, "%s: can't determine username\n", argv0 ); - exit( 1 ); + struct passwd* pw = getpwuid( getuid() ); + if ( pw == (struct passwd*) 0 ) { + (void) fprintf( stderr, "%s: can't determine username\n", argv0 ); + exit( 1 ); } - username = pw->pw_name; + username = pw->pw_name; #else /* DO_GETPWUID */ - (void) fprintf( stderr, "%s: can't determine username\n", argv0 ); - exit( 1 ); + (void) fprintf( stderr, "%s: can't determine username\n", argv0 ); + exit( 1 ); #endif /* DO_GETPWUID */ } - - if ( gethostname( hostname, sizeof(hostname) - 1 ) < 0 ) - show_error( "gethostname" ); - - if ( fake_from == (char*) 0 ) (void) snprintf( from, sizeof(from), "%s@%s", username, hostname ); - else + } else { if ( strchr( fake_from, '@' ) == (char*) 0 ) (void) snprintf( from, sizeof(from), "%s@%s", fake_from, hostname ); else (void) snprintf( from, sizeof(from), "%s", fake_from ); + } + + if ( gethostname( hostname, sizeof(hostname) - 1 ) < 0 ) + show_error( "gethostname" ); + /* Strip off any angle brackets in the from address. */ while ( from[0] == '<' )
make
mkdir -p /usr/local/skel/usr/sbin/
cp mini_sendmail /usr/local/skel/usr/sbin/
cd /usr/local/skel/usr/sbin/
ln -s mini_sendmail sendmail
Test
Create a couple of virtual hosts and see whether PHP runs. If all goes well, ps auxf output should look very similar to:
root 25761 0.0 1.7 138304 36360 ? Ss 10:56 0:00 /usr/sbin/apache2 -k start root 25762 0.0 0.0 4096 580 ? S 10:56 0:00 \_ /bin/sh -c /var/www/ispcp/engine/ispcp-apache-logger -e root 25764 0.0 0.7 62624 15424 ? S 10:56 0:00 | \_ /usr/bin/perl /var/www/ispcp/engine/ispcp-apache-logger -e root 25763 0.0 0.0 4096 576 ? S 10:56 0:00 \_ /bin/sh -c /var/www/ispcp/engine/ispcp-apache-logger root 25765 0.0 0.7 62624 15452 ? S 10:56 0:00 | \_ /usr/bin/perl /var/www/ispcp/engine/ispcp-apache-logger www-data 25766 0.0 1.6 138004 34416 ? S 10:56 0:00 \_ /usr/sbin/apache2 -k start www-data 25767 0.0 1.6 138004 34440 ? S 10:56 0:00 \_ /usr/sbin/fcgi-pm -k start vu2000 25768 0.0 0.4 147120 8936 ? Ss 10:56 0:00 | \_ /usr/bin/php5-cgi vu2000 25826 0.0 0.7 155448 15332 ? S 10:56 0:00 | | \_ /usr/bin/php5-cgi vu2000 25827 0.0 0.7 155192 14984 ? S 10:56 0:00 | | \_ /usr/bin/php5-cgi vu2007 11282 0.0 0.3 124356 7968 ? Ss 15:16 0:00 | \_ /usr/bin/php5-cgi vu2007 11284 0.1 0.5 129676 11560 ? S 15:16 0:00 | | \_ /usr/bin/php5-cgi vu2007 11285 0.1 0.5 131340 11756 ? S 15:16 0:00 | | \_ /usr/bin/php5-cgi vu2058 11292 1.0 0.3 124356 7972 ? Ss 15:17 0:00 | \_ /usr/bin/php5-cgi vu2058 11294 13.0 1.3 145292 27836 ? S 15:17 0:00 | \_ /usr/bin/php5-cgi vu2058 11295 0.0 0.1 124356 3512 ? S 15:17 0:00 | \_ /usr/bin/php5-cgi www-data 25769 0.0 1.8 428520 38416 ? Sl 10:56 0:00 \_ /usr/sbin/apache2 -k start www-data 25778 0.0 1.8 428356 38008 ? Sl 10:56 0:00 \_ /usr/sbin/apache2 -k start
Useful debugging tools include strace'ing the FastCGI Process Manager (PID 25767 in output above) if the vhost won't startup:
strace -ff -p 25767
Also, check the output in /tmp/suexec.log. Strace'ing the vhost's own processes is useful if they start up, but aren't outputting what you want to see (eg. for vu2058 in output above):
strace -ff -p 11292 -p 11294 -p 11295