142 lines
5.6 KiB
Markdown
142 lines
5.6 KiB
Markdown
# Optimize-the-IDOL-archive-queue-for-EU8_686074695
|
|
## Introduction
|
|
|
|
The IDOL archive queue on EU8 experiences a significant backlog. To address this issue, use a script that identifies the presently executing index job as the starting point and the latest index job at that moment as the end point for the IDOL content server. Then, cancel all index jobs within this range for the **Device** record type across all tenants.
|
|
|
|
Because this script selectively deletes specific device types, it can only differentiate index record types one by one, and then decide whether to perform the deletion operation. While each deletion request executes swiftly, the queue's index has surged to 770,000 as of Nov 10, 2023, implying that the overall execution might be time-consuming. Please plan the execution time accordingly.
|
|
|
|
## Required steps
|
|
|
|
1\. Log in to the bastion node.
|
|
|
|
2\. Run the following command for the **smarta-sawarc-con-0** pod:
|
|
|
|
```
|
|
kubectl exec -it smarta-sawarc-con-0 -n <NS> -c smarta-sawarc-con bash
|
|
```
|
|
|
|
3\. Under the **/var/data** folder, create a file named **delete\_device\_index\_job.sh** with the following content and assign it the execution permission.
|
|
|
|
```
|
|
#!/bin/bash
|
|
|
|
suite_admin_user="<userName>"
|
|
suite_admin_password="<password>"
|
|
|
|
suite_admin_token=""
|
|
get_token_time=""
|
|
device_index_num=0
|
|
device_entity_num=0
|
|
total_index_num=0
|
|
total_entity_num=0
|
|
|
|
hostname=$(hostname)
|
|
|
|
function delete_device() {
|
|
start_index_job=$(tail -n 200 /opt/content/logs/index.log | grep "Done Indexing IndexID" | cut -d "=" -f 2 | tail -n 1)
|
|
end_index_job=$(get_end_index_job)
|
|
echo "start_index_job: $start_index_job"
|
|
echo "end_index_job: $end_index_job"
|
|
for (( index_job_num = $start_index_job; index_job_num <= $end_index_job; index_job_num ++ ))
|
|
do
|
|
current_date_sec=$(date +%s)
|
|
sec=\`expr $current_date_sec - $get_token_time\`
|
|
if [ $sec -gt 120 ]
|
|
then
|
|
refresh_suite_admin_token
|
|
fi
|
|
if [ -e "status/${index_job_num}.data" ]
|
|
then
|
|
device_num=$(cat status/${index_job_num}.data | grep "ESS-DOCUMENT-TYPE=" | grep 'ESS-DOCUMENT-TYPE="Device"' | wc -l)
|
|
total_num=$(cat status/${index_job_num}.data | grep "ESS-DOCUMENT-TYPE=" | wc -l)
|
|
|
|
device_entity_num=\`expr $device_entity_num + $device_num\`
|
|
|
|
total_entity_num=\`expr $total_entity_num + $total_num\`
|
|
|
|
total_index_num=\`expr $total_index_num + 1\`
|
|
if [ $device_num -gt 0 -a $device_num -eq $total_num ]
|
|
then
|
|
device_index_num=\`expr $device_index_num + 1\`
|
|
cancel_index_in_content $index_job_num $device_num
|
|
device_index_percentage=$(printf "%d%%" $(($device_index_num*100/$total_index_num)))
|
|
device_entity_percentage=$(printf "%d%%" $(($device_entity_num*100/$total_entity_num)))
|
|
|
|
echo "index jobs -- device/total: $device_index_num:$total_index_num = $device_index_percentage"
|
|
echo "entity counts -- device/total: $device_entity_num:$total_entity_num = $device_entity_percentage"
|
|
echo ""
|
|
fi
|
|
|
|
|
|
fi
|
|
echo "${index_job_num}" > /tmp/last_canceled_index_job_num.txt
|
|
done
|
|
}
|
|
|
|
# base64url encode
|
|
function base64url_encode {
|
|
(if [ -z "$1" ]; then cat -; else echo -n "$1"; fi) |
|
|
openssl base64 -e -A |
|
|
sed s/\\+/-/g |
|
|
sed s/\\//_/g |
|
|
sed -E s/=+$//
|
|
}
|
|
|
|
# base64url decode
|
|
function base64url_decode {
|
|
INPUT=$(if [ -z "$1" ]; then echo -n $(cat -); else echo -n "$1"; fi)
|
|
MOD=$(($(echo -n "$INPUT" | wc -c) % 4))
|
|
PADDING=$(if [ $MOD -eq 2 ]; then echo -n '=='; elif [ $MOD -eq 3 ]; then echo -n '=' ; fi)
|
|
echo -n "$INPUT$PADDING" |
|
|
sed s/-/+/g |
|
|
sed s/_/\\//g |
|
|
openssl base64 -d -A
|
|
}
|
|
|
|
function cancel_index_in_content() {
|
|
cancel_cmd="https://${hostname}:1443/action=indexerGetStatus&ResponseFormat=simplejson&IndexAction=Cancel&index=$1"
|
|
base64_command=$(base64url_encode ${cancel_cmd})
|
|
response=$(curl -kSs "https://smarta-installer-svc:8443/itom-sma-smarta-mgmt/urest/v1.1/runIDOLCmd" \
|
|
--header 'Content-Type: application/x-www-form-urlencoded' \
|
|
--header "Cookie: BO_AUTH_TOKEN=${suite_admin_token}" \
|
|
--data-raw "cmdurl=${base64_command}")
|
|
response_status=$(echo $response | jq -r .idolresult.response | jq .autnresponse.responsedata.item[].description)
|
|
|
|
echo "Try to cancel index job id: $1, contains Device entity change count: $2, status: ${response_status}"
|
|
}
|
|
function get_end_index_job() {
|
|
end_index_cmd="https://${hostname}:1443/action=indexerGetStatus&MaxResults=1&ResponseFormat=simplejson"
|
|
base64_command=$(base64url_encode ${end_index_cmd})
|
|
response=$(curl -kSs "https://smarta-installer-svc:8443/itom-sma-smarta-mgmt/urest/v1.1/runIDOLCmd" \
|
|
--header 'Content-Type: application/x-www-form-urlencoded' \
|
|
--header "Cookie: BO_AUTH_TOKEN=${suite_admin_token}" \
|
|
--data-raw "cmdurl=${base64_command}")
|
|
end_index=$(echo $response | jq -r .idolresult.response | jq -r .autnresponse.responsedata.item[].id)
|
|
|
|
echo $end_index
|
|
}
|
|
|
|
function refresh_suite_admin_token() {
|
|
suite_admin_token=$(curl -kSs -X POST "https://itom-bo-login-svc:8443/bo/rest/auth/token" \
|
|
--header 'Content-Type: application/json' \
|
|
--data-raw "{
|
|
\"username\": \"${suite_admin_user}\",
|
|
\"password\": \"${suite_admin_password}\"
|
|
}")
|
|
get_token_time=$(date +%s)
|
|
echo "refresh_suite_admin_token"
|
|
}
|
|
|
|
refresh_suite_admin_token
|
|
delete_device
|
|
```
|
|
|
|
where:
|
|
|
|
- - userName: suite admin user name.
|
|
- password: password of your suite admin.
|
|
|
|
4\. Run this script: **./delete\_device\_index\_job.sh**
|
|
|
|
5\. Repeat steps 2 through 5 for the **smarta-sawarc-con-a-0** pod.
|