59 lines
2.2 KiB
Bash
59 lines
2.2 KiB
Bash
#!/bin/bash
|
|
|
|
# Application Insights JSON data Webservice and write into a SQL-Lite DB
|
|
# 2020-10-22 joe.tretter@kcc.com - Initial creation
|
|
|
|
cd `dirname "$0"`
|
|
mkdir -p inputdata tmp
|
|
. ./config.sh
|
|
|
|
timestamp=$(date +%Y%m%d%H%M%S)
|
|
|
|
if [[ "$1" == "" ]]; then
|
|
filename=inputdata/exp_${timestamp}.json
|
|
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
|
|
|
|
echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data"
|
|
# There is a hard 5000 record limit when using this API, can be lifted by switching to the Analytics API
|
|
# (https://dev.applicationinsights.io/documentation/Using-the-API/Query) which would have a different output format.
|
|
# For the current time, I believe we are not running 5K requests per day. And if we did, we can still run the script more often....
|
|
curl -o ${filename} -H "X-Api-Key: $ApiKey" "${Url}"
|
|
else
|
|
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
|
|
echo "$(date +%Y%m%d%H%M%S): reprocess file"
|
|
filename=$1
|
|
fi
|
|
|
|
# Thinking about it, the whole lot below could probably be done directly in JQ much faster and easier -- I'll maybe re-work that at some point.
|
|
echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile"
|
|
singleObjJSON=
|
|
objNestLevel=0 # used to determine the end over ever Object "record" in the input JSON.
|
|
cat ${filename} | jq '.value[]' | while read x; do
|
|
# echo -n "_"$x"_"
|
|
if [[ "${x}" == *"{"* ]]; then
|
|
let objNestLevel=objNestLevel+1
|
|
# echo -n ">>>>>"
|
|
fi
|
|
if [[ "${x}" == *"}"* ]]; then
|
|
let objNestLevel=objNestLevel-1
|
|
# echo -n "<<<<<"
|
|
fi
|
|
# echo :${objNestLevel}:
|
|
|
|
singleObjJSON=${singleObjJSON}$x
|
|
if [[ 0 -eq $objNestLevel ]]; then
|
|
insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));"
|
|
echo $insertSQL >> tmp/fillStage.${timestamp}.sql
|
|
echo -n "."
|
|
singleObjJSON=
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "$(date +%Y%m%d%H%M%S): - Load into DB"
|
|
cat prepareStage.sql tmp/fillStage.${timestamp}.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
|
|
#cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
|
|
|
|
echo "$(date +%Y%m%d%H%M%S): END"
|
|
|