Initial Checkin

.gitignore

@@ -0,0 +1,4 @@
+exp_*.json
+/inputdata
+/*.sqlite3
+*.stackdump

mergeData.sql

@@ -0,0 +1,3 @@
+select 'Merging Stage into requests';
+insert into requests select * from stage where not exists (select 1 from requests where requests.id=stage.id);
+

prepareStage.sql

@@ -0,0 +1,5 @@
+select 'Preparing Stage';
+drop table if exists stage;
+create table stage (id text, timestamp datetime, name text, success boolean, resultcode int, duration double, performancebucket string, userid string);
+create table if not exists requests (id text, timestamp datetime, name text, success boolean, resultcode int, duration double, performancebucket string, userid string);
+select 'Filling Stage';

recordAppInsights.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+cd `dirname "$0"`
+mkdir -p inputdata
+
+if [[ "$1" == "" ]]; then
+	filename=inputdata/exp_$(date +%Y%m%d%H%M%S).json
+	echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
+
+	echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data"
+	curl -o ${filename} -H "X-Api-Key: nwh7mwsnawjzopnhqmhf15ha37x38dtmwihvdbbk" 'https://api.applicationinsights.io/v1/apps/acd05e9b-705f-4c89-8eb0-fc5035913ac7/events/requests?timespan=P7D&$top=500000'
+else 
+	echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
+	echo "$(date +%Y%m%d%H%M%S): reprocess file"
+	filename=$1
+fi
+
+echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile"
+rm -f fillStage.sql
+singleObjJSON=
+declare -gA fields
+objNestLevel=0
+cat ${filename} | jq '.value[]' | while read x; do 
+#	echo -n "_"$x"_"
+	if [[ "${x}" == *"{"* ]]; then
+		let objNestLevel=objNestLevel+1
+#		echo -n ">>>>>"
+	fi
+	if [[ "${x}" == *"}"* ]]; then
+		let objNestLevel=objNestLevel-1
+#		echo -n "<<<<<"
+	fi
+#	echo :${objNestLevel}:
+	
+	singleObjJSON=${singleObjJSON}$x
+	if [[ 0 -eq $objNestLevel ]]; then
+		insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));"
+		echo $insertSQL >> fillStage.sql
+		echo -n "."
+		singleObjJSON=
+	fi
+done
+
+echo
+echo "$(date +%Y%m%d%H%M%S): - Load into DB"
+cat prepareStage.sql fillStage.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
+cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
+
+echo "$(date +%Y%m%d%H%M%S): END"
+

stats.sql

@@ -0,0 +1,2 @@
+select 'stage', count(*) from stage;
+select 'requests ', count(*) from requests;