nexus/knowledgebase/csd-wiki/ICSD/Optimize-the-IDOL-archive-queue-for-EU8_686074695.md

Optimize-the-IDOL-archive-queue-for-EU8_686074695

Introduction

The IDOL archive queue on EU8 experiences a significant backlog. To address this issue, use a script that identifies the presently executing index job as the starting point and the latest index job at that moment as the end point for the IDOL content server. Then, cancel all index jobs within this range for the Device record type across all tenants.

Because this script selectively deletes specific device types, it can only differentiate index record types one by one, and then decide whether to perform the deletion operation. While each deletion request executes swiftly, the queue's index has surged to 770,000 as of Nov 10, 2023, implying that the overall execution might be time-consuming. Please plan the execution time accordingly.

Required steps

1. Log in to the bastion node.

2. Run the following command for the smarta-sawarc-con-0 pod:

kubectl exec -it smarta-sawarc-con-0 -n <NS> -c smarta-sawarc-con bash

3. Under the /var/data folder, create a file named delete_device_index_job.sh with the following content and assign it the execution permission.

#!/bin/bash

suite_admin_user="<userName>"
suite_admin_password="<password>"

suite_admin_token=""
get_token_time=""
device_index_num=0
device_entity_num=0
total_index_num=0
total_entity_num=0

hostname=$(hostname)

function delete_device() {
    start_index_job=$(tail -n 200 /opt/content/logs/index.log | grep "Done Indexing IndexID" | cut -d "=" -f 2 | tail -n 1)
    end_index_job=$(get_end_index_job)
    echo "start_index_job: $start_index_job"
    echo "end_index_job: $end_index_job"
    for (( index_job_num = $start_index_job; index_job_num <= $end_index_job; index_job_num ++ ))
    do
        current_date_sec=$(date +%s)
        sec=\`expr $current_date_sec - $get_token_time\`
        if [ $sec -gt 120 ]
        then
            refresh_suite_admin_token
        fi
        if [ -e "status/${index_job_num}.data" ]
        then
            device_num=$(cat status/${index_job_num}.data | grep "ESS-DOCUMENT-TYPE=" | grep 'ESS-DOCUMENT-TYPE="Device"' | wc -l)
            total_num=$(cat status/${index_job_num}.data | grep "ESS-DOCUMENT-TYPE=" | wc -l)

            device_entity_num=\`expr $device_entity_num + $device_num\`

            total_entity_num=\`expr $total_entity_num + $total_num\`

            total_index_num=\`expr $total_index_num + 1\`
            if [ $device_num -gt 0 -a $device_num -eq $total_num ]
            then
                device_index_num=\`expr $device_index_num + 1\`
                cancel_index_in_content $index_job_num $device_num
                device_index_percentage=$(printf "%d%%" $(($device_index_num*100/$total_index_num)))
                device_entity_percentage=$(printf "%d%%" $(($device_entity_num*100/$total_entity_num)))

                echo "index jobs -- device/total: $device_index_num:$total_index_num = $device_index_percentage"
                echo "entity counts -- device/total: $device_entity_num:$total_entity_num = $device_entity_percentage"
                echo ""
            fi

            
        fi
        echo "${index_job_num}" > /tmp/last_canceled_index_job_num.txt
    done
}

# base64url encode
function base64url_encode {
  (if [ -z "$1" ]; then cat -; else echo -n "$1"; fi) |
    openssl base64 -e -A |
      sed s/\\+/-/g |
      sed s/\\//_/g |
      sed -E s/=+$//
}

# base64url decode
function base64url_decode {
  INPUT=$(if [ -z "$1" ]; then echo -n $(cat -); else echo -n "$1"; fi)
  MOD=$(($(echo -n "$INPUT" | wc -c) % 4))
  PADDING=$(if [ $MOD -eq 2 ]; then echo -n '=='; elif [ $MOD -eq 3 ]; then echo -n '=' ; fi)
  echo -n "$INPUT$PADDING" |
    sed s/-/+/g |
    sed s/_/\\//g |
    openssl base64 -d -A
}

function cancel_index_in_content() {
    cancel_cmd="https://${hostname}:1443/action=indexerGetStatus&ResponseFormat=simplejson&IndexAction=Cancel&index=$1"
    base64_command=$(base64url_encode ${cancel_cmd})
    response=$(curl -kSs "https://smarta-installer-svc:8443/itom-sma-smarta-mgmt/urest/v1.1/runIDOLCmd" \
    --header 'Content-Type: application/x-www-form-urlencoded' \
    --header "Cookie: BO_AUTH_TOKEN=${suite_admin_token}" \
    --data-raw "cmdurl=${base64_command}")
    response_status=$(echo $response | jq  -r .idolresult.response | jq .autnresponse.responsedata.item[].description)

    echo "Try to cancel index job id: $1, contains Device entity change count: $2, status: ${response_status}"
}
function get_end_index_job() {
    end_index_cmd="https://${hostname}:1443/action=indexerGetStatus&MaxResults=1&ResponseFormat=simplejson"
    base64_command=$(base64url_encode ${end_index_cmd})
    response=$(curl -kSs "https://smarta-installer-svc:8443/itom-sma-smarta-mgmt/urest/v1.1/runIDOLCmd" \
    --header 'Content-Type: application/x-www-form-urlencoded' \
    --header "Cookie: BO_AUTH_TOKEN=${suite_admin_token}" \
    --data-raw "cmdurl=${base64_command}")
    end_index=$(echo $response | jq  -r .idolresult.response | jq -r .autnresponse.responsedata.item[].id)

    echo $end_index
}

function refresh_suite_admin_token() {
    suite_admin_token=$(curl -kSs -X POST "https://itom-bo-login-svc:8443/bo/rest/auth/token" \
--header 'Content-Type: application/json' \
--data-raw "{
    \"username\": \"${suite_admin_user}\",
    \"password\": \"${suite_admin_password}\"
}")
    get_token_time=$(date +%s)
    echo "refresh_suite_admin_token"
}

refresh_suite_admin_token
delete_device

where:

• • userName: suite admin user name. - password: password of your suite admin.

4. Run this script: ./delete_device_index_job.sh

5. Repeat steps 2 through 5 for the smarta-sawarc-con-a-0 pod.