OptiTrainer/server/quoteDBs/continuousOptionTrainerLoad.sh

#!/bin/bash
while /bin/true; do
  (set -e
  find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do
    echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info 
    symbol=$(pbzip2 -dc "${singleFile}" | jq -e -r '.symbol' | tr -d '$')
    timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":")
    insertSQL=$(cat <<-EOF
	.timeout 10000;
	insert into OptionQuotes (
	  TimeStamp,
	  Symbol,
	  Data 
	) select 
	  datetime('${timeStamp}','utc'),
	  '${symbol}',
	  'test'
	where not exists (select 1 
	  from OptionQuotes 
	  where TimeStamp=datetime('${timeStamp}','utc') 
	  and Symbol='${symbol}'
	);
EOF
    )
    echo "$insertSQL" | sqlite3 consolidated.sqlite3
    touch -r "${singleFile}" lastRun.stamp 
  done) 2>&1 | systemd-cat -t "`basename $0`" -p debug 
  
  set +e
  echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info 
  sqlite3 consolidated.sqlite3 '.timeout 10000'\\n'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
      
  
  inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info 
  sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing...
done