joe/RecordAppInsights
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Misc. changes

Browse Source
This commit is contained in:
Joe Tretter
2020-10-23 17:49:37 +01:00
parent a460398d27
commit 0ccc453326
4 changed files with 23 additions and 5008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -2,3 +2,5 @@ exp_*.json
/inputdata /inputdata
/*.sqlite3 /*.sqlite3
*.stackdump *.stackdump
/config.sh
/fillStage.sql

4999
fillStage.sql
View File

File diff suppressed because it is too large Load Diff

4
mergeData.sql
View File

@@ -1,3 +1,7 @@
select 'Merging Stage into requests'; select 'Merging Stage into requests';
insert into requests select * from stage where not exists (select 1 from requests where requests.id=stage.id); insert into requests select * from stage where not exists (select 1 from requests where requests.id=stage.id);
drop table stage;
select 'Vacuuming the DB';
vacuum;

26
recordAppInsights.sh
View File

@@ -1,13 +1,23 @@
#!/bin/bash #!/bin/bash
# Application Insights JSON data Webservice and write into a SQL-Lite DB
# 2020-10-22 joe.tretter@kcc.com - Initial creation
cd `dirname "$0"` cd `dirname "$0"`
mkdir -p inputdata mkdir -p inputdata tmp
. ./config.sh
timestamp=$(date +%Y%m%d%H%M%S)
if [[ "$1" == "" ]]; then if [[ "$1" == "" ]]; then
filename=inputdata/exp_$(date +%Y%m%d%H%M%S).json filename=inputdata/exp_${timestamp}.json
echo "$(date +%Y%m%d%H%M%S): START [${filename}]" echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data" echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data"
curl -o ${filename} -H "X-Api-Key: nwh7mwsnawjzopnhqmhf15ha37x38dtmwihvdbbk" 'https://api.applicationinsights.io/v1/apps/acd05e9b-705f-4c89-8eb0-fc5035913ac7/events/requests?timespan=P7D&$top=500000' # There is a hard 5000 record limit when using this API, can be lifted by switching to the Analytics API
# (https://dev.applicationinsights.io/documentation/Using-the-API/Query) which would have a different output format.
# For the current time, I believe we are not running 5K requests per day. And if we did, we can still run the script more often....
curl -o ${filename} -H "X-Api-Key: $ApiKey" "${Url}"
else else
echo "$(date +%Y%m%d%H%M%S): START [${filename}]" echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
echo "$(date +%Y%m%d%H%M%S): reprocess file" echo "$(date +%Y%m%d%H%M%S): reprocess file"
@@ -15,10 +25,8 @@ else
fi fi
echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile" echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile"
rm -f fillStage.sql
singleObjJSON= singleObjJSON=
declare -gA fields objNestLevel=0 # used to determine the end over ever Object "record" in the input JSON.
objNestLevel=0
cat ${filename} | jq '.value[]' | while read x; do cat ${filename} | jq '.value[]' | while read x; do
# echo -n "_"$x"_" # echo -n "_"$x"_"
if [[ "${x}" == *"{"* ]]; then if [[ "${x}" == *"{"* ]]; then
@@ -34,7 +42,7 @@ cat ${filename} | jq '.value[]' | while read x; do
singleObjJSON=${singleObjJSON}$x singleObjJSON=${singleObjJSON}$x
if [[ 0 -eq $objNestLevel ]]; then if [[ 0 -eq $objNestLevel ]]; then
insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));" insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));"
echo $insertSQL >> fillStage.sql echo $insertSQL >> tmp/fillStage.${timestamp}.sql
echo -n "." echo -n "."
singleObjJSON= singleObjJSON=
fi fi
@@ -42,8 +50,8 @@ done
echo echo
echo "$(date +%Y%m%d%H%M%S): - Load into DB" echo "$(date +%Y%m%d%H%M%S): - Load into DB"
cat prepareStage.sql fillStage.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3 cat prepareStage.sql tmp/fillStage.${timestamp}.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/ #cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
echo "$(date +%Y%m%d%H%M%S): END" echo "$(date +%Y%m%d%H%M%S): END"