Initial checkin

README.md

@@ -0,0 +1,71 @@
+# Charls Schwab Quote recording bot
+
+## Expiry  
+* Access token expires in 30 Min.  
+* Refresh token expires in ?  
+  
+## File - Descriptions  
+### **`appCredentials.dat`**
+* Base64 encoded App Credentials - `"{app key}:{secret}"`  
+* Manually maintained  
+  
+### **`appKey.dat`**
+* only the application's AppKey  
+* Manually maintained  
+  
+#### **`allSymbols.json`**
+* List of all Symbols to get Data for  
+* Manually maintained  
+
+### **`access_token.dat`**
+* Current Access Token  
+* written by `getNewAccessToken.sh`
+  
+### **`accessTokenReqResp.json`**
+* Full JSON response of the last Access Token request  
+* written by `getNewAccessToken.sh`
+
+ ### **`refresh_token.dat`**
+* Current refresh token  
+* written by `getNewRefreshToken.sh`
+  
+### **`refreshTokenReqResp.json`**
+* Full JSON response of the last Refresh Token request  
+* written by `getNewRefreshToken.sh` 
+ 
+### **`codeReceiveServer.py`**
+* Python HTTPS web server to receive and output the Authentication code from the Schwab Website callback  
+* This web server only runs temporarily and ends after receiving a request
+  
+### **`cert.pem`**
+* self signed certificate for the temporary HTTPS server that receives the code.  
+* Manually maintained with:  
+    `openssl req -new -x509 -keyout cert.pem -out cert.pem -days 365 -nodes`
+	*Answer all questions hitting enter.*  
+
+## **`data/`**  
+* Folder containing date sub-folders for the output data  
+* Created and maintained by `recordSingleSymbolSingleRun.sh`
+  
+### **`getNewAccessToken.sh`**
+* Gets a new access Token and saves it to file `access_token.dat`
+  
+### **`getNewRefreshToken.sh`**
+* Gets a new refresh Token and saves it to file `refresh_token.dat`
+  
+### **`recordAllSymbolsSingleRun.sh`**
+* Runs `recordSingleSymbolSingleRun.sh` for every Symbol in file `allSymbols.json`
+  
+### **`recordQuoteSingleSymbolSingleRun.sh`**
+* requests stock quote data for a single symbol and writes output into the `data/*` Folder structure.  
+
+### **`recordOptionChainSingleSymbolSingleRun.sh`**
+* requests option chain data for a single symbol and writes output into the `data/*` Folder structure.  
+  
+
+# NEW CHANGES, NEED TO DOCUMENT
+jq <<< '{ "numberOfContracts":'$(( $(jq '.numberOfContracts' < ITM.json) +  $(jq '.numberOfContracts' < OTM.json) ))' }' > numberOfContracts.tmp
+jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ITM.json  > CommonDetail.tmp
+jq '. | map_values(select(type=="object"))' < ITM.json  > ITMo.tmp
+jq '. | map_values(select(type=="object"))' < OTM.json  > OTMo.tmp
+jq -s '.[0] * .[1] * .[2] * .[3]' numberOfContracts.tmp CommonDetail.tmp ITMo.tmp OTMo.tmp | less

allSymbols.json.example

@@ -0,0 +1,101 @@
+[
+    {
+        "Symbol": "AAPL"
+    },
+    {
+        "Symbol": "ACN"
+    },
+    {
+        "Symbol": "AMZN"
+    },
+    {
+        "Symbol": "BAC"
+    },
+    {
+        "Symbol": "CMG"
+    },
+    {
+        "Symbol": "CRM"
+    },
+    {
+        "Symbol": "CSCO"
+    },
+    {
+        "Symbol": "DIS"
+    },
+    {
+        "Symbol": "EDV"
+    },
+    {
+        "Symbol": "EFA"
+    },
+    {
+        "Symbol": "EWW"
+    },
+    {
+        "Symbol": "FSLR"
+    },
+    {
+        "Symbol": "GME"
+    },
+    {
+        "Symbol": "GOOG"
+    },
+    {
+        "Symbol": "HD"
+    },
+    {
+        "Symbol": "INTC"
+    },
+    {
+        "Symbol": "IWM"
+    },
+    {
+        "Symbol": "KMB"
+    },
+    {
+        "Symbol": "KO"
+    },
+    {
+        "Symbol": "LVS"
+    },
+    {
+        "Symbol": "MCD"
+    },
+    {
+        "Symbol": "META"
+    },
+    {
+        "Symbol": "NFLX"
+    },
+    {
+        "Symbol": "PM"
+    },
+    {
+        "Symbol": "PZZA"
+    },
+    {
+        "Symbol": "QQQ"
+    },
+    {
+        "Symbol": "SLV"
+    },
+    {
+        "Symbol": "SPY"
+    },
+    {
+        "Symbol": "TLT"
+    },
+    {
+        "Symbol": "$VIX"
+    },
+    {
+        "Symbol": "VXX"
+    },
+    {
+        "Symbol": "WMT"
+    },
+    {
+        "Symbol": "$XSP"
+    }
+]

codeReceiveServer.py

@@ -0,0 +1,30 @@
+#!/bin/python
+
+import http.server, ssl, socketserver
+import urllib.parse
+
+context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+context.load_cert_chain("cert.pem") # PUT YOUR cert.pem HERE
+server_address = ("0.0.0.0", 443) # CHANGE THIS IP & PORT
+#handler = http.server.SimpleHTTPRequestHandler
+
+#class MyHandler(http.server.SimpleHTTPRequestHandler):
+class MyHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        parsed_url = urllib.parse.urlparse(self.path)
+        code = urllib.parse.parse_qs(parsed_url.query).get('code', [None])[0]
+        if code:
+            print(code)
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            self.wfile.write("OK.".encode("utf-8"))
+            self.wfile.flush()
+            self.wfile.close()
+            quit()
+        else:
+            print("Invalid or missing authorization code")
+
+with socketserver.TCPServer(server_address, MyHandler) as httpd:
+    httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
+    httpd.serve_forever()

getNewAccessToken.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#set -x
+curl -s -X POST https://api.schwabapi.com/v1/oauth/token \
+  -H "Authorization: Basic $(<appCredentials.dat)" \
+  -H 'Content-Type: application/x-www-form-urlencoded' \
+  -d "grant_type=refresh_token&refresh_token=$(<refresh_token.dat)" > temp/accessTokenReqResp.$$.json
+jq -e -r .access_token temp/accessTokenReqResp.$$.json > access_token.dat
+if [[ $? -ne 0 ]]; then
+  errMsg="ERROR: GETTING THE ACCESS TOKEN FAILED! $(<temp/accessTokenReqResp.$$.json)"
+  systemd-cat -t "$0" -p err <<< ${errMsg}
+  >&2 echo "${errMsg}"
+  curl -d "" -X POST "https://deadswitch.kawomi.com/api/v1?topic=trade.schwab.error.getNewAccessToken&secs=1&email=joe@kawomi.com"
+fi
+#echo Access Token: $(<access_token.dat)
+#jq -r .refresh_token ./accessTokenReqResp.json > refresh_token.dat
+#echo Refresh Token: $(<refresh_token.dat)
+#rm ./refreshTokenReqResp.json
+

getNewRefreshToken.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#set -x
+myIP=$(ip a | grep 'inet.*192.168.10' | awk -F "[ /]*" '{print $3}')
+curl -s "https://ddns.kawomi.com/?domain=cstradebotcallback&address=${myIP}"
+echo
+
+echo OPEN URL: "https://api.schwabapi.com/v1/oauth/authorize?client_id=$(<appKey.dat)&redirect_uri=https://cstradebotcallback.ddns.kawomi.com/"
+echo
+theCode=$(sudo ./codeReceiveServer.py)
+echo Code is $theCode
+
+curl -s -X POST "https://api.schwabapi.com/v1/oauth/token" \
+  -H "Authorization: Basic $(<appCredentials.dat)" \
+  -H 'Content-Type: application/x-www-form-urlencoded' \
+  -d "grant_type=authorization_code&code=${theCode}&redirect_uri=https://cstradebotcallback.ddns.kawomi.com/" > temp/refreshTokenReqResp.$$.json
+jq -r .refresh_token temp/refreshTokenReqResp.$$.json > refresh_token.dat
+echo Refresh Token: $(<refresh_token.dat)
+

recordAllSymbolsSingleRun.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#set -x
+
+systemd-cat -t "$0" -p info <<< "Started"
+
+./getNewAccessToken.sh
+
+jq -r '.[].Symbol' allSymbols.json | sort | while read theSymbol; do
+  # I am not interested in detail quotes at the moment
+  #./recordQuoteSingleSymbolSingleRun.sh ${theSymbol}
+  #if [[ $? -ne 0 ]]; then
+  #  touch temp/hadErrors.$$.tmp
+  #fi
+
+  if [[ '$XSP' = ${theSymbol} ]]; then
+    ./recordOptionChainSingleSymbolSpecialRun.sh ${theSymbol}
+  else
+    ./recordOptionChainSingleSymbolSingleRun.sh ${theSymbol}
+    if [[ $? -ne 0 ]]; then
+       touch temp/hadErrors.$$.tmp
+
+#      Working around error in $XSP with special handling
+#      if [[ '$XSP' != ${theSymbol} ]]; then
+#        touch temp/hadErrors.$$.tmp
+#      fi
+    fi
+  fi
+done
+
+if [[ -e temp/hadErrors.$$.tmp ]]; then
+  rm temp/hadErrors.$$.tmp
+  curl -d "" -X POST "https://deadswitch.kawomi.com/api/v1?topic=trade.schwab.error.recordAllSymbolsSingleRun&secs=1&email=joe@kawomi.com"
+fi
+
+systemd-cat -t "$0" -p info <<< "Finished"

recordOptionChainSingleSymbolSingleRun.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#set -x
+symbol=$1
+outDir=data/$(date +%Y-%m-%d)
+
+mkdir -p "${outDir}"
+mkdir -p temp
+
+curl -s -X GET "https://api.schwabapi.com/marketdata/v1/chains?symbol=${symbol}" \
+  -H "Authorization: Bearer $(<access_token.dat)" > temp/optionChain.${symbol}.tmp.$$.json
+
+jqQuery=".symbol"
+symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.tmp.$$.json)
+if [[ "${symbol}" != "${symbolCheck}" ]]; then
+  errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.tmp.$$.json_"
+  systemd-cat -t "$0 $1" -p err <<< ${errMsg}
+  >&2 echo "${errMsg}"
+  exit 2
+else
+  optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json
+  #echo writing to file "${outDir}/${optionChainFileName}"
+  pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2"
+  rm temp/optionChain.${symbol}.tmp.$$.json
+fi
+
+

recordOptionChainSingleSymbolSpecialRun.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#set -x
+# This is for big chains, that cause the Schwab API to error out (like $XSP)
+# I add the range parameter (ITM/NTM/OTM) so that I get less data per request
+#
+symbol=$1
+outDir=data/$(date +%Y-%m-%d)
+
+mkdir -p "${outDir}"
+mkdir -p temp
+
+for range in ITM OTM NTM; do
+  curl -s -X GET "https://api.schwabapi.com/marketdata/v1/chains?symbol=${symbol}&range=${range}" \
+    -H "Authorization: Bearer $(<access_token.dat)" > temp/optionChain.${symbol}.${range}.tmp.$$.json
+
+  jqQuery=".symbol"
+  symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.${range}.tmp.$$.json)
+  if [[ "${symbol}" != "${symbolCheck}" ]]; then
+    errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.${range}.tmp.$$.json_"
+    systemd-cat -t "$0 $1" -p err <<< ${errMsg}
+    >&2 echo "${errMsg}"
+    exit 2
+  fi
+
+  optionChainPartFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.${range}.json
+  pbzip2 -9 -c < temp/optionChain.${symbol}.${range}.tmp.$$.json > "${outDir}/${optionChainPartFileName}.bz2"
+done
+
+###Paste together the parts, the NTM part, I don't think I need, but I will save it just in case..
+# sum up the number of Contracts
+
+ITMtempFile=temp/optionChain.${symbol}.ITM.tmp.$$.json
+OTMtempFile=temp/optionChain.${symbol}.OTM.tmp.$$.json
+jq <<< '{ "glued":true, "numberOfContracts":'$(( $(jq '.numberOfContracts' < ${ITMtempFile}) +  $(jq '.numberOfContracts' < ${OTMtempFile}) ))' }' > temp/numberOfContracts.$$.tmp
+jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ${ITMtempFile} > temp/CommonDetail.$$.tmp
+jq '. | map_values(select(type=="object"))' < ${ITMtempFile}  > ${ITMtempFile}.objOnly
+jq '. | map_values(select(type=="object"))' < ${OTMtempFile}  > ${OTMtempFile}.objOnly
+jq -s '.[0] * .[1] * .[2] * .[3]' temp/numberOfContracts.$$.tmp temp/CommonDetail.$$.tmp ${ITMtempFile}.objOnly ${OTMtempFile}.objOnly > temp/optionChain.${symbol}.tmp.$$.json
+
+optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json
+pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2"
+
+rm temp/*$$*

recordQuoteSingleSymbolSingleRun.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#set -x
+symbol=$1
+outDir=data/$(date +%Y-%m-%d)
+
+mkdir -p "${outDir}"
+mkdir -p temp
+
+curl -s -X GET "https://api.schwabapi.com/marketdata/v1/${symbol}/quotes?fields=quote%2Creference" \
+  -H "Authorization: Bearer $(<access_token.dat)" > temp/quote.${symbol}.tmp.$$.json
+
+jqQuery=".\"${symbol}\".symbol"
+symbolCheck=$(jq -r ${jqQuery} < temp/quote.${symbol}.tmp.$$.json)
+if [[ "${symbol}" != "${symbolCheck}" ]]; then
+  errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/quote.${symbol}.tmp.$$.json_"
+  systemd-cat -t "$0 $1" -p err <<< ${errMsg}
+  >&2 echo "${errMsg}"
+  exit 2
+else
+  quoteFileName=$(date "+%Y-%m-%d %H_%M_%S")_quote.${symbol}.json
+  #echo writing to file "${outDir}/${quoteFileName}"
+  pbzip2 -9 -c < temp/quote.${symbol}.tmp.$$.json > "${outDir}/${quoteFileName}.bz2"
+  rm temp/quote.${symbol}.tmp.$$.json
+fi
+
+