Robert de Bock

Een shell script om network throughput te meten op Linux machines.

Hier is een shell script om te zien hoeveel (kilo-, mega-, giga- of terra-) bytes de netwerk kaart langs komt. De output ziet er zo uit:

$ ./network-traffic.sh --help
Usage: ./network-traffic.sh [-i INTERFACE] [-s INTERVAL] [-c COUNT]

-i INTERFACE
    The interface to monitor, default is eth0.
-s INTERVAL
    The time to wait in seconds between measurements, default is 3 seconds.
-c COUNT
    The number of times to measure, default is 10 times.
$ ./network-traffic.sh        
Monitoring eth0 every 3 seconds. (RXbyte total = 706 Mb TXbytes total = 1 Gb)
RXbytes = 104 b TXbytes = 194 b
RXbytes = 80 b TXbytes = 188 b
RXbytes = 52 b TXbytes = 146 b
RXbytes = 689 b TXbytes = 8 Kb
RXbytes = 52 b TXbytes = 146 b
RXbytes = 52 b TXbytes = 146 b
RXbytes = 52 b TXbytes = 146 b
RXbytes = 52 b TXbytes = 146 b
RXbytes = 4 Kb TXbytes = 4 Kb
RXbytes = 716 b TXbytes = 5 Kb

Hier is het scripts:

#!/bin/sh

usage(){
echo "Usage: $0 [-i INTERFACE] [-s INTERVAL] [-c COUNT]"
echo
echo "-i INTERFACE"
echo "    The interface to monitor, default is eth0."
echo "-s INTERVAL"
echo "    The time to wait in seconds between measurements, default is 3 seconds."
echo "-c COUNT"
echo "    The number of times to measure, default is 10 times."
exit 3
}

readargs(){
while [ "$#" -gt 0 ] ; do
  case "$1" in
   -i)
    if [ "$2" ] ; then
     interface="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -s)
    if [ "$2" ] ; then
     sleep="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -c)
    if [ "$2" ] ; then
     counter="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   *)
    echo "Unknown option $1."
    echo
    shift
    usage
   ;;
  esac
done
}

checkargs(){
if [ ! "$interface" ] ; then
  interface="eth0"
fi
if [ ! "$sleep" ] ; then
  sleep="3"
fi
if [ ! "$counter" ] ; then
  counter="10"
fi
}

printrxbytes(){
/sbin/ifconfig "$interface" | grep "RX bytes" | cut -d: -f2 | awk '{ print $1 }'
}

printtxbytes(){
/sbin/ifconfig "$interface" | grep "TX bytes" | cut -d: -f3 | awk '{ print $1 }'
}

bytestohumanreadable(){
multiplier="0"
number="$1"
while [ "$number" -ge 1024 ] ; do
  multiplier=$(($multiplier+1))
  number=$(($number/1024))
done
case "$multiplier" in
  1)
   echo "$number Kb"
  ;;
  2)
   echo "$number Mb"
  ;;
  3)
   echo "$number Gb"
  ;;
  4)
   echo "$number Tb"
  ;;
  *)
   echo "$1 b"
  ;;
esac
}
 
printresults(){
while [ "$counter" -ge 0 ] ; do
  counter=$(($counter - 1))
  if [ "$rxbytes" ] ; then
   oldrxbytes="$rxbytes"
   oldtxbytes="$txbytes"
  fi
  rxbytes=$(printrxbytes)
  txbytes=$(printtxbytes)
  if [ "$oldrxbytes" -a "$rxbytes" -a "$oldtxbytes" -a "$txbytes" ] ; then
   echo "RXbytes = $(bytestohumanreadable $(($rxbytes - $oldrxbytes))) TXbytes = $(bytestohumanreadable $(($txbytes - $oldtxbytes)))"
  else
   echo "Monitoring $interface every $sleep seconds. (RXbyte total = $(bytestohumanreadable $rxbytes) TXbytes total = $(bytestohumanreadable $txbytes))"
  fi
  sleep "$sleep"
done
}

readargs "[email protected]"
checkargs
printresults

Apple Mac OS X als syslog server

Hier is een korte howto die je helpt je Mac OS X machine logs te ontvangen van apparaten, zoals de Apple Airport Extreme. Er zijn wat howto's online te vinden, maar ik denk dat er wat verandert is sinds Mac OS X 10.5, dus die howto's werken niet perfect.

Verander de syslogd configuratie

# echo "local0.notice /var/log/airport.log" >> /etc/syslog.conf

Touch het logfile

# touch /var/log/airport.log

Verander de syslogd startup procedure

Aan het einde van het file, uncomment het gedeelte dat remote logging accepteert.

# cat /System/Library/LaunchDaemons/com.apple.syslogd.plist
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>Label</key>
    <string>com.apple.syslogd</string>
    <key>OnDemand</key>
    <false/>
    <key>ProgramArguments</key>
    <array>
<!--
Un-comment the following lines to run syslogd with a sandbox profile.
Sandbox profiles restrict processes from performing unauthorized
operations; so it may be necessary to update the profile
(/usr/share/sandbox/syslogd.sb) if any changes are made to the syslog
configuration (/etc/syslog.conf).
-->
<!--
<string>/usr/bin/sandbox-exec</string>
<string>-f</string>
<string>/usr/share/sandbox/syslogd.sb</string>
-->
<string>/usr/sbin/syslogd</string>
    </array>
<key>MachServices</key>
<dict>
<key>com.apple.system.logger</key>
<true/>
</dict>
<key>Sockets</key>
<dict>
<key>AppleSystemLogger</key>
<dict>
<key>SockPathName</key>
<string>/var/run/asl_input</string>
<key>SockPathMode</key>
<integer>438</integer>
</dict>
<key>BSDSystemLogger</key>
<dict>
<key>SockPathName</key>
<string>/var/run/syslog</string>
<key>SockType</key>
<string>dgram</string>
<key>SockPathMode</key>
<integer>438</integer>
</dict>
<!--
Un-comment the following lines to enable the network syslog protocol listener.
-->
<key>NetworkListener</key>
<dict>
<key>SockServiceName</key>
<string>syslog</string>
<key>SockType</key>
<string>dgram</string>
</dict>
</dict>
</dict>
</plist>

Herstart syslogd

# launchctl unload /System/Library/LaunchDaemons/com.apple.syslogd.plist
# launchctl load /System/Library/LaunchDaemons/com.apple.syslogd.plist

Open de firewall

Ga naar de System Preferences, klik Security, open het Firewall tabje en klik de +. Selecteer het file /usr/bin/syslog.
Als je niet in staat bent om /usr te openen, kun je deze hack proberen in de terminal:

$ cd
$ ln -s /usr/bin
$ ln -s /usr/sbin

Nu staan de files in je homedirectory ./bin/syslog en ./sbin/syslogd

Configureer de apparaten

Vertel nu aan alle apparaten waarvan je logs wilt zien (zoals de Apple Airport Extreme) dat ze de logs kunnen dumpen op het IP adres van je Mac OS X machine.

Bekijk het resultaat

Je kunt de applicatie "Console" openen of vanaf een terminal dit commando draaien: "tail -f /var/log/airport.log".

Gebruik Putty en een HTTP proxy om overal met ssh te verbinden, door firewalls heen.

Zit je ooit in de situatie (bijvoorbeeld je werk waar Windows gebruikt wordt) waar je naar jouw machine (bijvoorbeeld je Linux/UNIX machine thuis) zou willen verbinden via SSH, maar de firewall regels laten dat niet toe? Misschien kun je de http-proxy server gebruiken om toch te verbinden.

Let wel op, deze technische truc is misshien wel mogelijk, maar het kan zijn dat het niet toegestaan is. Bespreek het eerst eens met iemand in je organisatie.

Hier zijn de ingrediënten:

  1. Een Windows machines die je gebruikt als client.
  2. Een Linux/UNIX machine die je gebruikt als (ssh) server.
  3. Download Putty of Portable Putty op je client.
  4. Zoek uit welke proxy jouw organisatie gebruikt op je client.
  5. Configureer Putty om die proxy te gebruiken op je client.

Welke proxy gebruikt jouw organisatie?
Er zijn een paar opties beschreven, van makkelijk naar moeilijk, kies er één. Kijk maar welke het beste werkt voor jou.

  1. Open Internet Explorer, klik Tools Internet Options.... Ga naar het tabblad Connections en klik LAN Settings.... In het Proxy server. gebied, zie je de proxy server en poort. Zo niet, ga door naar de volgende mogelijkheid.
  2. Op je windows machine, open een goede website en klik Start of zoiets, dan Run... en tik cmd in. Je krijgt nu een half kapotte terminal te zien. Tik netstat -rn en zoek naar ESTABLISHED verbindingen op poort 8080 of poort 3128. Het IP van de proxy server zie je staan in de derde kolom. De regel die ik zocht ziet er zo uit: TCP 192.168.1.2:2210 192.168.1.1:3128 ESTABLISHED. Het IP 192.168.1.1 is mijn proxy server, 3128 de poort.

Configureer Putty om deze proxy te gebruiken.
Nu dat je de proxy server en poort hebt, kun je Putty configureren.

  1. Open Putty en tik de Host Name waarmee je wilt verbinden. (De Linux/UNIX machine)
  2. Open het plusje voor Connections en kies Proxy. Tik de Proxy hostname en Port in.
  3. Kies nu Open. Als het goed is krijg je een password prompt te zien.

Gefeliciteerd, je bent alsnog verbonden!

Moving a single Drupal instalation into a multisite environment.

If you'd like to move a single installation of Drupal into a multisite environment, use these steps, and replace example.com for your websites name. In this example, Drupal is installed in /var/www/drupal/ .

1. Change the webserver configuration.

This one is obvious, the website was first pointing to an individual installation of Drupal, it needs to be directed to the multi-site installation of Drupal.

2. Copy templates, settings.php and files into the multisite environment.

mkdir -p /var/www/drupal/sites/example.com/
cd /var/www/example.com
cp -Rip html/sites/all/themes html/sites/default/settings.php html/sites/default/files/ /var/www/drupal/sites/example.com/

3. Update the MySQL database with the new paths.

If you have used files (including images) on your website, the path needs to be updated. Earlier files were located on "sites/default/files/", but this will become "sites/example.com/files"

mysql -u root -pPaSsWoRd
USE examplecom;
UPDATE files SET filepath=REPLACE (filepath, 'default', 'example.com');
UPDATE files SET filepath=REPLACE ('filepath', 'images', 'sites/example.com/images') WHERE filepath REGEXP '^images.*';
QUIT;

4. Change the location of the icons for the selected theme.

Go to Administer - Site building - Themes - Your Theme "configure" and change the path to reflect the right one. Mostly this means changing the word "files/" to "sites/example.com/files/".

5. Restart the web server and clean up the old environment.

For Apache, that would be:

apachectl configtest
apachectl restart

Check the website, everything should work, maybe you have to reselect your template to make it look better. If all works well, remove the old code.

rm -R /var/www/example.com

Shrinking a filesystem with LVM

After an installation you might find some file systems are too large, they are almost empty. When you want to use that space for another file system, here are the steps you can take:
Imagine /opt is now 10 Gb, but 1 Gb would be sufficient.

  1. Check if the file system is in use. Using lsof /opt you will get a list of processes that currently use /opt. Stop these processes.
  2. Find out what device is used for /opt with df -h /opt or mount. In my example, I found /dev/mapper/VolGroup/opt hold files on /opt.
  3. Unmount the filesystem, using umount /opt
  4. Resize the filesystem using resize2fs /dev/mapper/VolGroup/opt 1G. This frees the "right" part of the disk that LVM will un-allocate in a moment. All data from the file system is on the "left hand side".
  5. Run lvreduce -L 1G /dev/mapper/VolGroup-opt to shrink the logical volume. (It might warn you that you need to run e2fsck -f /dev/mapper/VolGroup-opt before you can continue.
  6. Remount the filesystem with a command as mount /opt.

For /opt or any other filesystem that can easily be freed from open file handles, the above procedure works fine, but for "busy" filesystems, like /, /var, /usr, and so on, you'd have boot the machine without mounting filesystems. One way to do this is using the installation CD and starting up the "rescue" environment.

Forwarding one port to another

Here is a very simple stick to forward a TCP port from your local workstation to another host. Can be easy to use for debugging purposes:

mkfifo pipe ; cat pipe | nc -l 8080 | nc google.com 80 > pipe ; rm pipe

What this one does:
1) Create a fifo (First in First out) file. This is a very simple type of file, you can put stuff in there with an output redirect (>) and get stuff out there with cat for example. It acts as a temporary buffer.
2) Open that newly created pipe. Anything that gets in, will be printed. (and forwarded in this example to "nc")
3) Open a listening port on your local workstation, listening on port 8080.
4) Open a connection to google.com, on port 80.
5) Send al the output to the earlier created pipe.
6) Remove the pipe when done.

Have a look the netcat homepage, it's a great tool!

Creating an RPM of some binary

We've covered this topic before in this story about creating an RPM from a shell script, but this information might help you better understand how to create an RPM.

So; you've found a piece of software that has no RPM? (Or; your manager tells you to install a piece of software that the development department created.)

Normally you'd use ./configure ; make ; make install, here is how to put that all in an RPM.

Prepare your rpm building environment: (DO THIS AS A USER!)

$ sudo yum install rpm-build
$ mkdir -p RPMBUILD/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
$ echo "%_topdir /home/username/RPMBUILD" >> .rpmmacros

Now copy the software into that newly create structure.

$ cp software.tar.gz RPMBUILD/SOURCES/

And now create a "spec file" for the software. This basically explains rpmbuild how to make the software and what to put in the RPM. This is the most "tweakable" step and might require quite some time to get right. Put this into /home/username/RPMBUILD/SPECS/software.spec:

Name: software
Version: 0.23
Release: 1
Summary: Custom software to run enterprise servers.

Group: Applications/Internet
License: GPLv2
URL: http://meinit.nl/
Source0: %{name}-%{version}.tar.gz
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root

%description
This software runs all enterprise software as a daemon. It's been developed by Me in IT consultancy.

%prep
%setup -q

%build
make

%install
mkdir -p $RPM_BUILD_ROOT/usr/local/bin
install software $RPM_BUILD_ROOT/usr/local/bin/software

%files
%defattr(-,root,root)
%doc README
/usr/local/bin/software

%clean
rm -rf $RPM_BUILD_ROOT

%changelog
* Tue Jun 15 2010 Robert de Bock <[email protected]> - 0.23-1
- Initial build

Good to know; the %install refers to the temporary environment that rpm will create when building this RPM. The %files section refers to what will end up in the RPM. They should correspond; you can't %install a whole bunch of files and only include a few in the $files part. (rpmbuild will display the missing files.

The group can be any line out of /usr/share/doc/rpm-*/GROUPS

So; you are prepared, run this command to so if you got everything correct:

$ rpmbuild -ba software.spec

When it finally builds, you'll find the rpm in /home/username/RPMBUILD/RPMS/$arch/software-0.23-1.$arch.rpm

Setting up iSCSI (target/server and initiator/client) on RHEL

It's quite easy to setup an iSCSI environment on Red Hat Enterprise Linux. Try this easy setup to get a better understanding of iSCSI.

Ingredients:

  1. Two (virtual) machines, a server 172.16.0.1 and a client 172.16.0.2.
  2. Access to the "RHEL Cluster-Storage" channel on Red Hat Network.

N.B. SELinux must be disabled when using this recipe, iptables tcp port 3260 must be opened on the server.

On the server execute these commands to setup a 100 Mb iSCSI target. This target can later be mounted on the client(s).

# yum install scsi-target-utils
# cat /etc/tgt/targets.conf
<target iqn.2010-04.nl.meinit:node1.target1>
backing-store /iscsi1.img
initiator-address 172.16.0.2
</target>
# dd if=/dev/zero of=/iscsi1.img bs=1024 count=102400
# chkconfig tgtd on
# service tgtd start

Now on (all) client(s) follow these steps. (Please pay attention that only one client was give access in the configuration example above; 172.16.0.2.)

# yum install iscsi-initiator-utils

Start iscsi daemon.

# service iscsi start

To see what IQNs are available, run:

# iscsiadm -m discovery -t sendtargets -p 172.16.0.1

The result is a list of IQN(s) available. This discovery is a mandatory step of connecting to the iSCSI target.

Login to the iSCSI target:

# iscsiadm -m node -T iqn.2010-04.nl.meinit:node1.target1 -p 172.16.0.1 -l

If that all works, you have new SCSI devices available, check dmesg and start iscsi at boot time:

# chkconfig iscsi on

In this example the iSCSI target does not have a filesystem. Create it on the client and mount it at boot time:

# fdisk /dev/sda
# mkfs.ext3 /dev/sda1
# echo "/dev/sda1 /mnt ext3 defaults,_netdev 0 0" >> /etc/fstab

You are done, but these commands are quite useful when connecting to an unknown iSCSI device.

To see more about the IQN:

# iscsiadm -m node -T iqn.2010-04.nl.meinit:node1.target1 -p 172.16.0.1

Restore hidden files with Apple Mac OS X Time Machine

Apple's Time Machine works great, but restoring hidden files (files that start with a dot, like .ssh, .bashrc or .Trash) is difficult, but possible!

Time machine uses the settings as used by the Finder. So first step is to change Finders behaviour, to show hidden files. Execute this command (as a regular user) from within the Terminal.

$ defaults write com.apple.finder AppleShowAllFiles TRUE
$ killall Finder

Now you should be able to see extra files in the finder, like this:

Now start Time Machine and scroll back to the date you were sure a file existed.

Restore it and to hide all these (annoying) hidden files, revert to original Finder settings:

$ defaults write com.apple.finder AppleShowAllFiles FALSE
$ killall Finder

Nagios time check using SNMP

When you would like to retrieve the remotely configured time using SNMP and compare it to see how accurate the time is, here is a script to help you out.

This setup does not specifically require NTP to be running on the hosts that are checked, it just requires that the time is correct. Virtual machines for example are advised to have the appropriate "tools" installed to synchronize time. NTP is not desirable for virtual machines.

(Parts of the script are borrowed from http://spielwiese.la-evento.com/xelasblog/archives/27-SNMP-hrSystemDate.0-Datumsformat-anpassen.html)

This is the graph that is created:

The script:

#!/bin/sh

# Nagios plugin to report time difference as received via SNMP compared to the local time.
# Make sure the machine this script runs on (poller/nagios host) is using NTP.

usage() {
# This function is called when a user enters impossible values.
echo "Usage: $0 -H HOSTADDRESS [-C COMMUNITY] [-w WARNING] [-c CRITICAL] [-v VERSION]"
echo
echo " -H HOSTADDRESS"
echo "     The host to check, either IP address or a resolvable hostname."
echo " -C COMMUNITY"
echo "     The SNMP community to use, defaults to public."
echo " -v VERSION"
echo "     The SNMTP version to use, defaults to 2c."
echo " -w WARNING"
echo "     The amount of seconds from where warnings start. Defaults to 60."
echo " -c CRITICAL"
echo "     The amount of seconds from where criticals start. Defaults to 120."
exit 3
}

readargs() {
# This function reads what options and arguments were given on the
# command line.
while [ "$#" -gt 0 ] ; do
  case "$1" in
   -H)
    if [ "$2" ] ; then
     host="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -C)
    if [ "$2" ] ; then
     community="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -w)
    if [ "$2" ] ; then
     warning="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -c)
    if [ "$2" ] ; then
     critical="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   -v)
    if [ "$2" ] ; then
     version="$2"
     shift ; shift
    else
     echo "Missing a value for $1."
     echo
     shift
     usage
    fi
   ;;
   *)
    echo "Unknown option $1."
    echo
    shift
    usage
   ;;
  esac
done
}

checkvariables() {
# This function checks if all collected input is correct.
if [ ! "$host" ] ; then
  echo "Please specify a hostname or IP address."
  echo
  usage
fi
if [ ! "$community" ] ; then
  # The public community is used when a user did not enter a community.
  community="public"
fi
if [ ! "$version" ] ; then
  # Version 2c is used when a user did not enter a version.
  version="2c"
fi
if [ ! "$critical" ] ; then
  critical="120"
fi
if [ ! "$warning" ] ; then
  warning="60"
fi
}

getandprintresults() {
# This converts the date retreived from snmp to a unix time stamp.
rdatestring=$( snmpget -v $version -c $community $host HOST-RESOURCES-MIB::hrSystemDate.0 | gawk '{print $NF}' )

if [ ! "$rdatestring" ] ; then
  echo "Time difference could not be calculated; no time received."
  exit 3
fi

rdate=$( echo $rdatestring | gawk -F',' '{print $1}' )
rtime=$( echo $rdatestring | gawk -F',' '{print $2}' | gawk -F'.' '{print $1}' )
cldate=$( echo $rdate | gawk -F'-' '{printf("%4i",$1)}; {printf("%02i",$2)}; {printf("%02i",$3)};' )
cltime=$( echo $rtime | gawk -F':' '{printf("%02i",$1)}; {printf("%02i",$2)}; {printf(" %02i",$3)};' )
rdate_s=$( date -d "$cldate $cltime sec" +%s )
ldate_s=$(date +'%s')

# If the calculated difference is negative, make it positive again for comparison.
difference=$(($rdate_s - $ldate_s))
if [ "$difference" -lt 0 ] ; then
  positivedifference=$(($difference*-1))
else
  positivedifference=$difference
fi

if [ "$positivedifference" -gt "$critical" ] ; then
  echo "Time difference is more than $critical seconds: $difference|diff=$difference"
  exit 2
fi

if [ "$positivedifference" -gt "$warning" ] ; then
  echo "Time difference is more than $warning seconds: $difference|diff=$difference"
  exit 1
fi

echo "Time difference is less than $warning seconds: $difference|diff=$difference"
exit 0
}

# The calls to the different functions.
readargs "[email protected]"
checkvariables
getandprintresults

To implement it in Nagios, add these sniplets to nagios.cfg. (or any other applicable nagios file.)

The service for a group.

define service{
        hostgroup_name                  Servertype_Linux
        service_description             time
        _SERVICE_ID                     1856
        use                             SNMP-time
}

The service template.

define service{
        name                            SNMP-time
        service_description             time
        use                             generic-service
        check_command                   check_snmp_time!$_HOSTSNMPCOMMUNITY$!120!60
        max_check_attempts                      30
        normal_check_interval           5
        retry_check_interval            1
        notification_interval           0
        register                                0
}

The command.

define command{
        command_name                    check_snmp_time
        command_line                    $USER1$/check_snmp_time -H $HOSTADDRESS$ -C $ARG1$ -c $ARG2$ -w $ARG3$
}

Syndicate content