user  nginx nginx;
worker_processes  1;

error_log  /www/log/error-ssl.log;
pid        /var/run/nginx-ssl.pid;

events {
    worker_connections  16;
    }

http {

    include       mime.types;
    default_type  application/octet-stream;

    log_format  main '"$host" "$request" "[$time_local]" "$bytes_sent"';
    access_log  /www/log/access-ssl.log  main;

    ## General Options
    ignore_invalid_headers   on;
    sendfile                 on;
    server_tokens           off;

    gzip  on;
    keepalive_timeout  65;

 server {
        listen       443;
        server_name  linux-sysadmin.org;
        ssl on;
        ssl_certificate /www/ssl/linux-sysadmin.org.crt;
        ssl_certificate_key /www/ssl/linux-sysadmin.org.key.unsecure;
        ssl_client_certificate /opt/nginx/conf/ssl/ca.crt;
        ssl_verify_client       on;

        access_log  /www/log/linux-sysadmin.access-ssl.log  main;

        location / {
                fastcgi_pass unix:/www/app/web/run/serviceMgmt.socket;
                # necessary parameter
                fastcgi_param PATH_INFO $fastcgi_script_name;
                # to deal with POST requests
                fastcgi_param REQUEST_METHOD $request_method;
                fastcgi_param CONTENT_TYPE $content_type;
                fastcgi_param CONTENT_LENGTH $content_length;
                # necessary parameter for https
                fastcgi_param  HTTPS              on;

        
        }

        location /media {
        root /www/app/web; 

        }

    }
  }

#!/sbin/runscript                                                     

opts="${opts} stop start"

depend() {
        need net 
        use dns logger netmount
        after ntpd
}

DJANGO_SITE="serviceMgmt"
SITES_PATH=/www/app/web
RUNFILES_PATH=$SITES_PATH/run
RUN_AS=django
PYTHON_EGG_CACHE=/tmp

export PYTHON_EGG_CACHE

start() {
        ebegin "Starting django-fgci"
        chmod -R 777 $SITES_PATH
        if [ -f $RUNFILES_PATH/$DJANGO_SITE.pid ]; then
                ewarn "Django already running ($RUNFILES_PATH/$DJANGO_SITE.pid)"
        else
                start-stop-daemon --start \
                --pidfile $RUNFILES_PATH/$DJANGO_SITE.pid \
                --chuid $RUN_AS \
                --exec /usr/bin/python $SITES_PATH/$DJANGO_SITE/manage.py runfcgi \
                socket=$RUNFILES_PATH/$DJANGO_SITE.socket \
                pidfile=$RUNFILES_PATH/$DJANGO_SITE.pid
                chmod 777 $RUNFILES_PATH/$DJANGO_SITE.socket
                einfo "Django started"
        fi
        eend $?
}

stop() {
        ebegin "Stopping django-fgci"
        VERBOSE_NAME="$DJANGO_SITE"
        start-stop-daemon --stop \
        --pidfile $RUNFILES_PATH/$DJANGO_SITE.pid \
        && einfo "Django stopped" \
        || ewarn "Django not running"
        if [ -f $RUNFILES_PATH/$DJANGO_SITE.pid ]; then
                rm $RUNFILES_PATH/$DJANGO_SITE.pid
                rm $RUNFILES_PATH/$DJANGO_SITE.socket
        fi
        eend $?
}

Best way for NGINX logs rotation

I wanted rotate access logs every minute without HUP signal or reopen utility.
Why? Because, i wanted fluently write into logs and logs must contain access
for 1 minute. I didn’t want call reopen utility and then move file, because i moved
files from quick (heap) storage to slow storage.

Finally i did patch which you can find in tools section.

Installation:

1, download patch
2, download nginx 0.8.53
3, extract nginx and go to the src dir
4, call patch: patch  -p0 -i <path_to_dir_where_is_patch>/bbcdnTime0.8.53.patch
5, set in your nginx.conf something like this (keyword is $date_bbcdn) :

access_log  /logdir/access_$date_bbcdn.access.log  log;

and then you can see access logs in following format:
access_1293111000.access.log

And now you have new log every minute without any rotation script.

Description: Patch for rotating access logs every minute

Download: bbcdnTime0.8.53.patch

Installation:

1, download patch
2, download nginx 0.8.53
3, extract nginx and go to the src dir
4, call patch: patch  -p0 -i <path_to_dir_where_is_patch>/bbcdnTime0.8.53.patch
5, set in your nginx.conf something like this (keyword is $date_bbcdn) :

access_log  /logdir/access_$date_bbcdn.access.log  log;

and then you can see access logs in following format:
access_1293111000.access.log

1, Exit but leave content on screen

i often use less, but that’s horrible when you do exit and then

you will see only terminal without the content.

I found in manual function:

-X or –no-init
Disables sending the termcap initialization and deinitialization
strings to the terminal.  This is  sometimes  desirable  if  the
deinitialization  string does something unnecessary, like clear‐
ing the screen.

great function, now you can read text file and after exit (q button) you

can see the content of text file.

example:

less -X test.txt

That’s fine, you can do some alias for “less -X”, but when are you connected in new

server then is your new function of less out. So, what will we do with this?

Just easy use your .bashrc or .bash_profile config file and put into this file:

export LESS=X

and then you have perfect less cmd

2, read your file with less and edit at the moment

In the manual you can find function:

v  —   Invokes  an editor to edit the current file being viewed.  The editor is taken from the environment variable VISUAL if defined, or EDITOR if VISUAL is not defined, or defaults to “vi” if neither VISUAL nor EDITOR is defined.  See also the discussion of LESSEDIT under  the  section    on PROMPTS below.

Example:

read your file with less command and if you want edit this file just press “v”.

If you know more tips for less, tell me this please in discussion…

Create SW RAID from disks which were in HW RAID ARRAY

When i built my new server, i got disks from old server.

In old server was HW RAID array and the new server doesn’t supports HW RAID.

So, i boot sysrescue CD on server and when i built new raid array i saw this message:

mdadm: device /dev/sda not suitable for any style of array

Unfortunately systemrescue want create hw raid, because he see some flags in old disk.

$ dmsetup status

shows some hw arrays.

You can remove all hw raids with

$ dmsetup remove_all

but, after the restart system see only hw raid in degrade mode, because hw raid looks like one disk.

How i can repair it?

You must remove some blocks in the end of partition, where is info about HW RAID.

For example:

1, show your disk capacity in sectors

fdisk -lu /dev/sda

Disk /dev/sda5: 20.3 GB, 20341085184 bytes
255 heads, 63 sectors/track, 2472 cylinders, total 39728682 sectors
Units = sectors of 1 * 512 = 512 bytes

2, and then you can remove last 10MB blocks where are information about HW RAID

$ dd if=/dev/zero of=/dev/sda bs=512 seek=39718442

After build array and reboot system you don’t see any hw raid device in

$ dmsetup status

and then you will see nothing and problem with

mdadm: device /dev/sda not suitable for any style of array

is gone…

That’s all

How you can monitor your http traffic?

I was creating small python script which can analyse your http traffic via apache access log.

The name of my script is  apacheLogReport.

Result of script shows grouped report of network traffic sorted by hostname, referer and file. He is useful when you finding what makes your http traffic.

How to use this script:

At first you must edit your apache httpd.conf file.

Find you virtual host for monitoring and insert these lines:

TransferLog “|/rotatelogs /www/log/httpd-access-analyze.log 3600″
LogFormat “‘%h’ ‘%f’  ‘%{Referer}i\’ ‘%b’”

Attention: set your paths to rotatelogs and access log correctly!

So, the hardest work is behind us

Now, run a check result of script.

$chmod +x apacheLogReport.py

$./apacheLogReport.py –help
Usage: apacheLogReport.py [options] args

Options:
–version             show program’s version number and exit
-h, –help            show this help message and exit
-d, –description     Detail info
-f FILE, –file=FILE  Specify which file to analyze
-l LIMIT, –limit=LIMIT
Items limit (default 10)

$./apacheLogReport.py -f /www/log/httpd-access-analyze.log -l 20

example of result:

Download links:

apacheLogReport from this site

apacheLogReport form sourceforge.net

Do you have any idea for improvements?

Tell me please your opinion discussion.

Thank you!

Do you want log only specific request and not all http requests?

It’s easy with apache module setenvif_module

Your options (terms) for logging are:

• Remote_Host
• Remote_Addr
• Server_Addr
• Request_Method
• Request_Protocol
• Request_URI

for example i have some requests like this:

http://linux-sysadmin.org/cms/show_article/8001.html?query=2

http://linux-sysadmin.org/cms/show_categories/6025_1.html?query=5

I want log request which contain word article and number 8001

in httpd.conf:

SetEnvIf Request_URI (article) my_request
SetEnvIf Request_URI (8001) my_request

CustomLog “|/opt/apache/bin/rotatelogs /var/web/log/httpd-test-access.log 3600″ \

“%h \”%r\” %t \”%{Referer}i\”" env=my_request

That’s is easy, but what must i do if i want log request by query string?

Unfortunately SetEnvIf can’t work for this problem, but you can use mod_rewrite module

and combine it with SetEnvIf.

Example of use (i want log request where query string is ‘query=2′ + previous terms):

in httpd.conf:

RewriteEngine On

SetEnvIf Request_URI (article) my_request
SetEnvIf Request_URI (8001) my_request

RewriteCond %{QUERY_STRING}  (query=2)
RewriteRule  (.*)   $1 [E=my_request:yes]

CustomLog “|/opt/apache/bin/rotatelogs /var/web/log/httpd-test-access.log 3600″ \

“%h \”%r\” %t \”%{Referer}i\”" env=my_request

Easy?

I didn’t use special regexp. Of course, you can use very difficult regular (regexp) patterns

for find your specific request.

Tags: Apache Httpd, SetEnvIf, mod_rewrite, CustomLog, request

My favourite scripts for tuning mysql server variables are:

1, tuning-primer.sh

This script takes information from “SHOW STATUS LIKE…”
and “SHOW VARIABLES LIKE…” then attempts to produce sane recommendations
for tuning server variables. It is compatible with all versions of MySQL 3.23 – 5.1.

How to use:

a, download

b, edit tunnig-primer.sh via your favourite text editor and set path to your socket

at line 38

c, run with your favourite arguments (without arguments is the best choice):

#########################################################################
#                                                                       #
# Little known feature: 1st argument is execution mode                  #
#                                                                       #
#       Usage: ./tuning-primer.sh [ mode ]                              #
#                                                                       #
#       Available Modes:                                                #
#               all :           perform all checks (default)            #
#               prompt :        prompt for login credintials and socket #
#                               and execution mode                      #
#               mem, memory :   run checks for tunable options which    #
#                               effect memory usage                     #
#               disk, file :    run checks for options which effect     #
#                               i/o performance or file handle limits   #
#               innodb :        run InnoDB checks /* to be improved */  #
#               misc :          run checks for that don’t categorise    #
#                               well Slow Queries, Binary logs,         #
#                               Used Connections and Worker Threads     #
########################################################################

2, mysqltuner.pl

is similar like  tuning-primer.sh.

options for mysqltuner are:

Connection and Authentication
–host <hostname>    Connect to a remote host to perform tests (default: localhost)
–port <port>        Port to use for connection (default: 3306)
–user <username>    Username to use for authentication
–pass <password>    Password to use for authentication

Output Options:
–nogood             Remove OK responses
–nobad              Remove negative/suggestion responses
–noinfo             Remove informational responses
–nocolor            Don’t print output in color

Both are great MySQL High Performance Tuning Scripts.

I prefer tuning-primer.sh, because is written in bash.

Download links:

tuning-primer.sh

mysqltuner.pl

Mysql pager is great feature for filter your processlist and others results.

You can use your commands before displaying output to you.

Examples:

mysql> pager less

mysql> pager cat

ok, it was easy and without gain.

Better examples:

1, show only specific pattern in result

mysql> pager grep -i localhost
PAGER set to ‘grep -i localhost’
mysql> show processlist;
| 377113 | smith      | localhost       | test | Sleep   | 46622 |       | NULL             |
| 377115 | smith      | localhost       | test | Sleep   | 46348 |       | NULL             |
| 392250 | smith      | localhost       | test | Query   |     0 | NULL  | show processlist |
13 rows in set (0.00 sec)

2, count occurrence of pattern

mysql> pager (grep -i localhost | wc -l)
PAGER set to ‘(grep -i localhost | wc -l)’
mysql> show processlist;
3
13 rows in set (0.00 sec)

3, show queries in Sleep status which have time bigger then 40 seconds

mysql> pager (grep -i Sleep | awk ‘$12 > 40′)
PAGER set to ‘(grep -i Sleep | awk ‘$12 > 40′)’
mysql> show processlist;
| 377113 | smith      | localhost       | test | Sleep   | 46851 |       | NULL             |
| 377115 | smith      | localhost       | test | Sleep   | 46577 |       | NULL             |
| 392209 | smith | localhost | test | Sleep   |    91 |       | NULL             |
| 392233 | smith | localhost | test | Sleep   |   121 |       | NULL             |
| 392235 | smith | localhost | test | Sleep   |    61 |       | NULL             |
| 392236 | smith | localhost | test | Sleep   |    61 |       | NULL             |
13 rows in set (0.00 sec)

4, sort result by Command

mysql> pager sort -k 12
PAGER set to ‘sort -k 12′

5, sort result by user

mysql> pager sort -k 4
PAGER set to ‘sort -k 4′

6, sort result by host

mysql> pager sort -k 6
PAGER set to ‘sort -k 6′

7, sort by everything what you want

mysql> pager sort -k [your column]

8, show your specific user which have time bigger then 40 seconds

mysql> pager awk ‘$4 ~/smith/ && $12 > 40′
PAGER set to ‘awk ‘$4 ~/smith/ && $12 > 40”
mysql> show processlist;
| 392209 | smith | localhost| test | Sleep   |   109 |       | NULL             |
| 392210 | smith | localhost | test | Sleep   |    49 |       | NULL             |
| 392217 | smith | localhost | test | Sleep   |    49 |       | NULL             |
| 392232 | smith | localhost | test | Sleep   |    49 |       | NULL             |
| 392233 | smith | localhost | test | Sleep   |   139 |       | NULL             |
| 392235 | smith | localhost | test | Sleep   |    79 |       | NULL             |
| 392236 | smith | localhost| test | Sleep   |    79 |       | NULL             |

9, you can use your script, or everything else. Only your mind is the limit