using pbzip2 for parallelity and make the frontend compatible with old TDA and new Schwab symbol names for options (different separator)

server/quoteDBs/continuousOptionTrainerLoad.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+while /bin/true; do
+  (set -e
+  find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do
+    echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info 
+    symbol=$(pbzip2 -dc "${singleFile}" | jq -r '.symbol' | tr -d '$')
+    timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":")
+    insertSQL=$(cat <<EOF
+      insert into OptionQuotes (
+        TimeStamp,
+        Symbol,
+        Data 
+      ) select 
+          datetime('${timeStamp}','utc'),
+          '${symbol}',
+          '$(pbzip2 -dc "${singleFile}")'
+        where not exists (select 1 
+           from OptionQuotes 
+          where TimeStamp=datetime('${timeStamp}','utc') 
+            and Symbol='${symbol}'
+      );
+EOF
+    )
+    echo $insertSQL | sqlite3 consolidated.sqlite3
+    touch -r "${singleFile}" lastRun.stamp 
+  done) 2>&1 | systemd-cat -t "`basename $0`" -p debug 
+  
+  set +e
+  echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info 
+  sqlite3 consolidated.sqlite3 'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
+  
+  inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info 
+  sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing...
+done