From 886467327c015942932018adecc74d68da2a1c5e Mon Sep 17 00:00:00 2001 From: Joe Tretter Date: Thu, 19 Mar 2020 08:16:56 -0500 Subject: [PATCH] DB-processing, split initial and update --- .../quoteDBs/{fillBigDB.sh => fillBigDB_initial.sh} | 4 +++- server/quoteDBs/fillBigDB_update.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) rename server/quoteDBs/{fillBigDB.sh => fillBigDB_initial.sh} (88%) create mode 100644 server/quoteDBs/fillBigDB_update.sh diff --git a/server/quoteDBs/fillBigDB.sh b/server/quoteDBs/fillBigDB_initial.sh similarity index 88% rename from server/quoteDBs/fillBigDB.sh rename to server/quoteDBs/fillBigDB_initial.sh index 8d0c2c9..f8845bb 100644 --- a/server/quoteDBs/fillBigDB.sh +++ b/server/quoteDBs/fillBigDB_initial.sh @@ -1,7 +1,8 @@ #!/bin/bash -rm consolidated.sqlite3 +mkdir -p processedDBs +rm consolidated.sqlite3 sqlite3 consolidated.sqlite3 < createTable.sql ls quoteStore_202*.sqlite3.gz | while read db; do @@ -11,4 +12,5 @@ ls quoteStore_202*.sqlite3.gz | while read db; do sqlite3 /tmp/xx.sqlite3 '.dump OptionQuotes' | sqlite3 consolidated.sqlite3 echo deleting... rm /tmp/xx.sqlite3 + mv $db processedDBs done diff --git a/server/quoteDBs/fillBigDB_update.sh b/server/quoteDBs/fillBigDB_update.sh new file mode 100644 index 0000000..bae70bc --- /dev/null +++ b/server/quoteDBs/fillBigDB_update.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +mkdir -p processedDBs + +ls quoteStore_202*.sqlite3.gz | while read db; do + echo unpacking $db + gzip -dc $db >/tmp/xx.sqlite3 + echo processing... + sqlite3 /tmp/xx.sqlite3 '.dump OptionQuotes' | sqlite3 consolidated.sqlite3 + echo deleting... + rm /tmp/xx.sqlite3 + mv $db processedDBs +done