#!/bin/bash
# https://github.com/sklochkov/performance-tuner/blob/master/src/network-queues.sh
# https://github.com/torvalds/linux/blob/master/Documentation/networking/scaling.rst

# set_affinity=$1

# if [ -n "`lsb_release -i | grep -i "debian"`" ]
# then
#     apt-get install -y net-tools bc ethtool
# else
#     yum install -y net-tools bc ethtool
# fi

#安装mpstat 中断检查工具
if command -v mpstat &>/dev/null; then
    echo "mpstat is already installed on this system."
else
    echo "mpstat is not installed on this system. Downloading now..."
    sudo yum -y install sysstat
fi

# cpu核心数
cpus=$(grep -c processor /proc/cpuinfo)

str=""
for vth in $(ls /sys/devices/virtual/net/); do
    str="${vth}|${str}"
done
# 虚拟网卡列表，以|分割
vthstr=$(echo ${str} | sed 's/^\(.*\)|$/\1/g')

# 物理网卡列表，以空格分割
function get_interfaces() {
    interfaces=$(sed -n '3,$p' /proc/net/dev | awk -F ":" '{print $1}' | grep -Ev "${vthstr}" | sed 's/ //g')

    if [[ -z ${interfaces} ]]; then
        #获取默认路由网卡
        interfaces=$(/usr/sbin/ip route | grep default | head -n1 | grep -Eo 'dev \w+' | awk '{print $2}')
    fi
}
get_interfaces
#判断cpu是否存在中断
function RPS_cpus() {
    for k in $interfaces; do
        IP_num=$(ls /sys/class/net/$k/queues/ | grep rx | wc -l)
        i=$(($IP_num - 1))
        for j in $(seq 0 $i); do
            for e in $k; do
                CPUS=$(cat /sys/class/net/$e/queues/rx-${j}/rps_cpus | grep -v "f")
            done
        done
    done
}
RPS_cpus

#0914判断标准
#判断中断是否较高
mpstat=$(mpstat -P ALL 3 1 | grep Average | awk '{print $8}' | grep -v soft | awk '$1 > 30 {print $1}')

if [ -z "$mpstat" ] && [ -z "$CPUS" ]; then
    echo "CPU的soft正常"
    echo $mpstat
    echo $CPUS
    if [ -z "$CPUS" ]; then
        echo "中断已经执行"
        echo $CPUS
        exit 1
    else
        echo "没有执行中断脚本"
        echo $CPUS
    fi
else
    if [ -z "$mpstat" ]; then
        echo "CPU的soft超过30"
        echo $mpstat
    else
        if [ -z "$CPUS" ]; then
            echo "中断已经执行"
            echo $CPUS
            exit 1
        else
            echo "CPU的soft不正常,执行中断脚本"
            echo $CPUS
        fi
    fi
fi
# 物理网卡列表，以|分割
#interfaces_shugang=`sed -n '3,$p' /proc/net/dev | awk -F ":" '{print $1}' | grep -Ev "${vthstr}" | sed 's/ //g' |tr "\n" "|"  | sed 's/^\(.*\)|$/\1/g'`

filter_interfaces=""
for interface in ${interfaces}; do
    ip_down=$(ip add | grep -w "${interface}:" | grep -v 'NO-CARRIER')
    if [ -z "${ip_down}" ]; then
        echo "interface: ${interface} is down! remove it"
    else
        if [ -z ${filter_interfaces} ]; then
            filter_interfaces="${interface}"
        else
            filter_interfaces="${filter_interfaces} ${interface}"
        fi
    fi
    # echo -e "interfaces:\n${interfaces}"
    # echo "filter_interfaces:${filter_interfaces}"
done

#echo -e "interfaces:\n${interfaces}"
echo "filter_interfaces:${filter_interfaces}"

# 值覆盖
interfaces=${filter_interfaces}

#关闭irqbalance
# /bin/systemctl stop irqbalance.service

# eth接口
for interface in ${interfaces}; do

    #ethtool -l ${interface}
    n_combined_max=$(/sbin/ethtool -l ${interface} 2>/dev/null | grep -A4 max | grep Combined: | awk '{print $2}' | head -n1)
    if [[ ! ${n_combined_max} ]]; then
        echo ${interface}"不可设置队列"
        #开启irqbalance
        # /bin/systemctl start irqbalance.service
        # continue
    else
        #设置网卡队形
        /bin/systemctl stop irqbalance.service
        n_combined_current=$(/sbin/ethtool -l ${interface} | grep -A4 Current | grep Combined: | awk '{print $2}' | head -n1)
        n_combined_set=0
        n_tx_max=$(/sbin/ethtool -l ${interface} | grep -A4 max | grep TX: | awk '{print $2}' | head -n1)
        n_tx_current=$(/sbin/ethtool -l ${interface} | grep -A4 Current | grep TX: | awk '{print $2}' | head -n1)
        n_rx_max=$(/sbin/ethtool -l ${interface} | grep -A4 max | grep RX: | awk '{print $2}' | head -n1)
        n_rx_current=$(/sbin/ethtool -l ${interface} | grep -A4 Current | grep RX: | awk '{print $2}' | head -n1)
        n_tx_set=0
        n_rx_set=0

        if [[ ${n_combined_current} -gt 0 ]]; then

            #n_combined_set不受cpu限制
            # if [[ ${n_combined_max} -lt ${cpus} ]];then
            #     n_combined_set=${n_combined_max}
            # else
            #     n_combined_set=${cpus}
            # fi
            n_combined_set=${n_combined_max}

            if [[ ${n_combined_current} -lt ${n_combined_set} ]]; then
                echo "调整网卡队列数：${n_combined_current} -> ${n_combined_set}"
                /sbin/ethtool -L ${interface} combined ${n_combined_set}
            fi
            n_set=${n_combined_set}
        else
            echo "rx tx分别设置"
            if [[ ${n_tx_max} -lt ${cpus} ]]; then
                n_tx_set=${n_tx_max}
            else
                n_tx_set=${cpus}
            fi

            if [[ ${n_tx_current} -ne ${n_tx_set} ]]; then
                echo "调整网卡tx队列数：${n_tx_current} -> ${n_tx_set}"
                /sbin/ethtool -L ${interface} tx ${n_tx_set}
            fi
            # 调tx一样
            n_rx_set=$n_tx_set
            if [[ ${n_rx_current} -ne ${n_rx_set} ]]; then
                echo "调整网卡rx队列数：${n_rx_current} -> ${n_rx_set}"
                /sbin/ethtool -L ${interface} rx ${n_rx_set}
            fi
            n_set=${n_tx_set}
        fi

        #ethtool -g ${interface}
        n_tx_size_max=$(/sbin/ethtool -g ${interface} | grep -A4 max | grep TX: | awk '{print $2}' | head -n1)
        n_tx_size_current=$(/sbin/ethtool -g ${interface} | grep -A4 Current | grep TX: | awk '{print $2}' | head -n1)
        n_tx_size_set=0
        n_rx_size_max=$(/sbin/ethtool -g ${interface} | grep -A4 max | grep RX: | awk '{print $2}' | head -n1)
        n_rx_size_current=$(/sbin/ethtool -g ${interface} | grep -A4 Current | grep RX: | awk '{print $2}' | head -n1)

        if [[ ${n_tx_size_max} -lt 4096 ]]; then
            n_tx_size_set=${n_tx_size_max}
        else
            n_tx_size_set=4096
        fi

        if [[ ${n_tx_size_current} -ne ${n_tx_size_set} ]]; then
            echo "调整网卡tx size：：${n_tx_size_current} -> ${n_tx_size_set}"
            /sbin/ethtool -G ${interface} tx ${n_tx_size_set}
        fi
        # #调tx一半
        # n_rx_size_set=$[n_tx_size_set/2]
        # 调tx一样
        n_rx_size_set=$n_tx_size_set
        if [[ ${n_rx_size_current} -ne ${n_rx_size_set} ]]; then
            echo "调整网卡rx size：${n_rx_size_current} -> ${n_rx_size_set}"
            /sbin/ethtool -G ${interface} rx ${n_rx_size_set}
        fi

        #rps_sock_flow_entries
        rps_sock_flow_entries=$((n_tx_size_set * n_set))
        echo $rps_sock_flow_entries >/proc/sys/net/core/rps_sock_flow_entries
    fi

    if_pppoe_running=$(ps -ef | grep -i 'pppd' | grep -v grep | wc -l)

    # 中断数少于核心数
    # https://github.com/torvalds/linux/blob/master/Documentation/networking/scaling.rst#suggested-configuration-3

    # 跳过cpu0
    # if [ ${cpus} -lt 32 ]
    # then
    #     default_affinity="fffffe"
    # else
    #     default_affinity="fffffffe"
    # fi
    #不跳过cpu0

    default_affinity=$(python -c '''
import sys;import re;import sys;import os;from collections import OrderedDict;from subprocess import STDOUT, check_output, CalledProcessError;from sys import argv

def run_cmd(cmdstr):
    try:
        output = check_output(cmdstr, stderr=STDOUT, shell=True)
    except CalledProcessError as exc:
        output = exc.output
    return output


def fn_count_default_affinity():
    cups = run_cmd("grep -c processor /proc/cpuinfo").strip()
    cups=int(cups)
    if cups < 10:
        str_int2=int(cups)*"1"
    else:
        str_int2=(int(cups)-2)*"1"+"0"+"1"
    int16=hex(int(str_int2,2))
    str_f=int16.strip("0xLl")
    if  len(str_f) > 8 :
        list_tmp = list(str_f)
        list_tmp.insert(-8, ",")
        if len(str_f) > 16:
            list_tmp.insert(-17, ",")
            if len(str_f) > 24:
                list_tmp.insert(-26, ",")
                if len(str_f) > 32:
                    list_tmp.insert(-35, ",")
        print("".join(list_tmp))
    else :
        print(str_f)
fn_count_default_affinity()''')

    # if [[ ! $default_affinity  ]];then
    #     default_affinity=$(cat /root/default_affinity)
    # fi
    # echo ${default_affinity} > /root/default_affinity

    echo "==================================================================="
    echo "                网卡:     ${interface}              "
    echo "                cpu核数:     ${cpus}              "
    echo "                内存：" $(free -h | awk '/Mem/ {print $2}' | tr -cd '[0-9]')
    echo "                combined队列数:${n_combined_set}               "
    echo "               tx队列数: ${n_tx_set}"
    echo "               rx队列数: ${n_rx_set}"
    echo "               n_tx_size:     ${n_tx_size_set}               "
    echo "               n_rx_size:     ${n_rx_size_set}              "
    echo "                rps_flow_cnt: ${n_tx_size_set}               "
    echo "                rps_sock_flow_entries:    ${rps_sock_flow_entries}               "
    echo "                affinity: ${default_affinity}               "

    echo "==================================================================="

    #affinity自定义
    #default_affinity="fffffffb"

    q=($(grep -E "${interface}" /proc/interrupts | awk '{print $1}' | cut -d: -f 1 | grep -e '^[0-9]*$'))
    echo "${interface} 物理网卡中断号列表:${q[*]}"
    echo "${interface} 中断号数量:${#q[@]}"

    echo "绑定硬中断 开始"
    # 跳过cpu0
    k=1                  # k伴随中断号递增
    for i in ${q[*]}; do # i表示中断号
        echo ${k} >/proc/irq/$i/smp_affinity_list
        #echo "irq: ${i} cpu_list: `cat /proc/irq/${i}/smp_affinity_list`"
        k=$((${k} + 1))
        if [ ${k} -eq ${cpus} ]; then
            k=0
        fi
    done
    echo "绑定硬中断 结束"
    echo "========================================="

    echo "绑定软中断 开始"

    if [ ${if_pppoe_running} -gt 0 ]; then
        # 有ppp拨号 rps rfs都开

        echo "default_affinity: ${default_affinity}"
        RX=$(ls -ld /sys/class/net/${interface}/queues/rx* 2>/dev/null | wc -l)
        TX=$(ls -ld /sys/class/net/${interface}/queues/tx* 2>/dev/null | wc -l)
        if [ $RX -gt 0 ]; then

            for i in $(seq $RX); do
                echo ${default_affinity} >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus
                echo ${n_tx_size_set} >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt
                echo "/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus $(cat /sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus)"
                echo "/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt $(cat /sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt)"
            done
        fi
        if [ $TX -gt 0 ]; then
            for i in $(seq $TX); do
                echo "${default_affinity}" >/sys/class/net/${interface}/queues/tx-$(($i - 1))/xps_cpus
                #echo "/sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus `cat /sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus`"
            done
        fi
    elif [ ${#q[@]} -lt ${cpus} ]; then
        # 没有ppp拨号 rps开 rfs不开
        echo '中断号数目小于cpu数'

        RX=$(ls -ld /sys/class/net/${interface}/queues/rx* 2>/dev/null | wc -l)
        TX=$(ls -ld /sys/class/net/${interface}/queues/tx* 2>/dev/null | wc -l)
        if [ $RX -gt 0 ]; then
            for i in $(seq $RX); do
                echo ${default_affinity} >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus
                # brpc thread-model don't open rfs
                echo "0" >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt
                echo "/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus $(cat /sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus)"
                echo "/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt $(cat /sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt)"
            done
        fi
        if [ $TX -gt 0 ]; then
            for i in $(seq $TX); do
                echo "${default_affinity}" >/sys/class/net/${interface}/queues/tx-$(($i - 1))/xps_cpus
                #echo "/sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus `cat /sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus`"
            done
        fi
        # brpc thread-model don't open rfs
        echo "0" >/proc/sys/net/core/rps_sock_flow_entries
    else
        # close rps/rfs
        echo "0" >/proc/sys/net/core/rps_sock_flow_entries
        if [ -d /sys/class/net/${interface}/queues ]; then
            RX=$(ls -ld /sys/class/net/${interface}/queues/rx* 2>/dev/null | wc -l)
            TX=$(ls -ld /sys/class/net/${interface}/queues/tx* 2>/dev/null | wc -l)
            if [ $RX -gt 0 ]; then
                for i in $(seq $RX); do
                    echo "0" >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_cpus
                    echo "0" >/sys/class/net/${interface}/queues/rx-$(($i - 1))/rps_flow_cnt
                    #echo "/sys/class/net/${interface}/queues/rx-$[$i - 1]/rps_cpus `cat /sys/class/net/${interface}/queues/rx-$[$i - 1]/rps_cpus`"
                    #echo "/sys/class/net/${interface}/queues/rx-$[$i - 1]/rps_flow_cnt `cat /sys/class/net/${interface}/queues/rx-$[$i - 1]/rps_flow_cnt`"
                done
            fi
            if [ $TX -gt 0 ]; then
                for i in $(seq $TX); do
                    echo "0" >/sys/class/net/${interface}/queues/tx-$(($i - 1))/xps_cpus
                    #echo "/sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus `cat /sys/class/net/${interface}/queues/tx-$[$i - 1]/xps_cpus`"
                done
            fi
        fi
    fi
    echo "绑定软中断 结束"
    echo "========================================="
done
