LDAP scalability

The following scripts are suggested to provide a LDAP scalability. The basic idea is that LDAP lookups are flaky in the version we are using (client 2.0.27) and as a consequence, some lookups (including /usr/bin/id) fails in mass. Additionally, in our LDAP version, caching do not seem to work either.

The scripts below "dump" the entry list to a local file so that network lookups are un-necessary while LDAP is still used to centralized the account information. They were written and deployed at PDSF and passed as-is to the RCF team for consideration.

create_files in /etc/cron.hourly

#!/bin/sh

#
# This script dumps the user and group data from ldap and creates standard
# UNIX passwd and group files. This are propogated by cfengine to the compute nodes
#
# Shane Canon
# 2005

# Target destination
CFENGINE=/auto/config/nis
EXT="new"
HOST="-H ldaps://pdsfadmin02.nersc.gov/"

ldapsearch -x -L -L $HOST -b 'ou=people,o=ldapsvc,dc=nersc,dc=gov' > ldap-users.dump

# Did it work?
if [ $? -ne 0 ] ; then
echo "Error with ldap query"
exit -1
fi

LINES=`wc -l ldap-users.dump|awk '{print $1}'`
if [ $LINES -lt 100 ] ; then
echo "User file from LDAP to short...exiting"
exit -1
fi

ldapsearch -x -L -L $HOST -b 'ou=PosixGroup,o=ldapsvc,dc=nersc,dc=gov' > ldap-groups.dump

# Did it work?
if [ $? -ne 0 ] ; then
echo "Error with ldap query"
exit -1
fi

LINES=`wc -l ldap-groups.dump|awk '{print $1}'`
if [ $LINES -lt 100 ] ; then
echo "Group file from LDAP to short...exiting"
exit -1
fi

PASS="${CFENGINE}/passwd.ldap"
SHAD="${CFENGINE}/shadow.ldap"
PASSI="${CFENGINE}/passwd.ldap.int"
SHADI="${CFENGINE}/shadow.ldap.int"
GROUP="${CFENGINE}/group.ldap"
GSHAD="${CFENGINE}/gshadow.ldap"
# Temporary account
TPASS="$PASS.$EXT"
TSHAD="$SHAD.$EXT"
TPASSI="$PASSI.$EXT"
TSHADI="$SHADI.$EXT"
TGROUP="$GROUP.$EXT"
TGSHAD="$GSHAD.$EXT"

CAT="/bin/cat"
AWK="/bin/awk"
GREP="/bin/grep"
EGREP="/bin/egrep"
LN="/bin/ln"
RM="/bin/rm"
CP="/bin/cp"
SORT="/bin/sort"
CHOWN="/bin/chown"
CHMOD="/bin/chmod"
#
# Start the passwd file
# - Use the local passwd file for entries below 500
# - Ignore u70004 (its in LDAP already)
# - Ignore expired accounts
# - Sort the list by uid
$CAT /etc/passwd|$AWK -F: '{if ($3<500){print $0}}'| \
$GREP -v u70004| \
$GREP -v expired| \
$GREP -v disabled| \
$SORT -t: -n +2 > $TPASS
#
# Add in lsfadmin which is above 500
#
$GREP lsfadmin /etc/passwd >> $TPASS
$CAT /etc/passwd|$AWK -F: '{if ($4==4000){print $0}}' >> $TPASS

# This will be our new interactive passwd file
$CP $TPASS $TPASSI

# Let's create the shadow account for these entries
$CAT $TPASS|$AWK -F: '{print "^"$1":"}'|xargs -n 1 $EGREP /etc/shadow -e > $TSHAD

$CAT ldap-users.dump|./users.pl >> $TPASS

$CAT /etc/group|$AWK -F: '{if ($3<500){print $0}}'> $TGROUP

# Let's create the group shadow file for these entries
$CAT $TGROUP|$AWK -F: '{print $1":!!::"}' > $TGSHAD

$CAT ldap-groups.dump|./groups.pl >> $TGROUP


# Semi-safe copy
#
$LN -f $TPASS $PASS
$LN -f $TPASSI $PASSI
$LN -f $TGROUP $GROUP
$LN -f $TSHAD $SHAD
$LN -f $TGSHAD $GSHAD


$RM $TPASS
$RM $TPASSI
$RM $TGROUP
$RM $TSHAD
$RM $TGSHAD

$CHOWN bin.bin $SHAD
$CHOWN bin.bin $GSHAD
$CHMOD 640 $SHAD
$CHMOD 640 $GSHAD

Formatting scripts

Two add-on scripts are needed as follow

groups.pl

#!/usr/bin/perl
#
# This script convert from LDAP LDIFF format to a standard UNIX group format
#
# Shane Canon
# 2005
#


while(<>){
chop;
if (/^$/){
if ($dn=~/ou=PosixGroup,ou=pdsf,ou=Host,o=ldapsvc,dc=nersc,dc=gov/ ){
createEntry();
}
undef $attr{homeDirectory25};
undef $members;
}
else{
($field,$value)=split /: /;
if ($field eq 'dn'){
$dn=$value;
}
elsif($field eq 'memberUid'){
if (defined $members){
$members.=',';
}
$members.=$value;
}
else{
$attr{$field}=$value;
}
}
}
createEntry();
foreach $id (sort {$a <=> $b} keys %groups){
print $groups{$id}."\n";
}


sub createEntry{
if ( $attr{gidNumber}>100 ){
$line=sprintf "%s:x:%d:%s",
$attr{cn},$attr{gidNumber},$members;
$groups{$attr{gidNumber}}=$line;
}


}
#
# Example
#
#dn: cn=m144,ou=Groups,o=ldapsvc,dc=nersc,dc=gov
#cn: m144
#gidNumber: 4883
#objectClass: top
#objectClass: PosixGroup
#memberUid: fukazawa
#structuralObjectClass: PosixGroup
#creatorsName: uid=nimrepo,ou=people,dc=nersc,dc=gov
#createTimestamp: 20040629222732Z
#modifiersName: uid=nimrepo,ou=people,dc=nersc,dc=gov
#modifyTimestamp: 20040629222732Z
#

users.pl

#!/usr/bin/perl
#
# This script convert from LDAP LDIFF format to a standard UNIX passwd format
#
# Shane Canon
# 2005
#

$SysID="";
$hdID="homeDirectory".$SysID;
$shID="loginShell".$SysID;

open(LOG,">> /var/log/ldap.log");
while(<>){
chop;
if (/^$/){
if ($dn=~/people/ ){
createEntry();
}
undef $attr{$hdID};
}
else{
($field,$value)=split /: /;
if ($field eq 'dn'){
$dn=$value;
}
else{
$attr{$field}=$value;
}
}
}
createEntry();

foreach $id (sort {$a <=> $b} keys %users){
print $users{$id}."\n";
}


sub createEntry{
$id=$attr{uidNumber};
$dir=$attr{$hdID};
$dir=~s/\/auto\/u/\/u/;
$shell=$attr{$shID};
$shell=~s/\/bin\/csh/\/bin\/tcsh/;
if ( defined $attr{$hdID} ){
$line=sprintf "%s:x:%d:%d:%s:%s:%s",
$attr{uid},$attr{uidNumber},$attr{gidNumber},$attr{gecos},$dir,$shell;
print LOG "Warning: user id $id already exists! \n\t".$users{$id}."\n\t$line\n" if defined $users{$id};
$users{$attr{uidNumber}}=$line;
# print LOG "$id: $line\n";
}
}