Lots of re-structuring and handling of different data types

DBExtractor/DBExtractor.config.json

@@ -0,0 +1,14 @@
+{
+    "dbDriver":"mssql", "dbDriverComment":"either mssql (user/password) or mssql/msnodesqlv8 (trustedConnection)",
+    "processingType": "casrpslog",
+    "sourceTag": "CCERpsLog",
+    "serverURI": "http://localhost:2222",
+    "pollIntervalIdle": 10000,
+    "sqlConfig": {
+        "user": "casdbuser",
+        "password": "4KcLao2016!",
+        "server": "ustww2063\\CCE",
+        "database": "CAS110",
+        "port": 1433
+    }
+}

DBExtractor/DBExtractor.config.json.example

@@ -0,0 +1,25 @@
+{
+    "dbDriver":"mssql", "dbDriverComment":"either mssql (user/password) or mssql/msnodesqlv8 (trustedConnection)",
+    "processingType": "casrpslog",
+    "sourceTag": "CCERpsLog",
+    "serverURI": "http://localhost:2222",
+    "pollIntervalIdle": 10000,
+    "UP_sqlConfig": {
+        "user": "casdbuser",
+        "password": "4KcLao2016!",
+        "server": "ustww2063\\CCE",
+        "database": "CAS110",
+        "port": 1433
+    },
+    "TC_sqlConfig": {
+        "driver": "msnodesqlv8",
+        "server": "ust2cas04g1",
+        "database": "LAOCAS110QA",
+        "port": 1433,
+        "options": {
+            "trustedConnection": true,
+            "useUTC": true
+        }
+    }
+
+}

DBExtractor/DBExtractor.js

@@ -0,0 +1,108 @@
+const fs = require("fs");
+const config = JSON.parse(fs.readFileSync("DBExtractor.config.json", "utf8"));
+
+const sql = require(config.dbDriver);
+const request = require("request-promise");
+const winston = require("winston");
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => {
+            return `${info.timestamp} ${info.level}: ${info.message}`;
+        })
+    ),
+    transports: [
+        new winston.transports.Console(),
+        new winston.transports.File({ filename: './log/DBExtractorError.log', level: 'error' }),
+        new winston.transports.File({ filename: './log/Combined.log' })
+    ]
+});
+
+console.log(`DBExtractor Copyright (C) 2019  j.tretter@gmail.com
+    
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.`);
+
+logger.log("info", `DBExtractor v0.1 Starting`);
+
+
+logger.log("debug", "Config" + JSON.stringify(config))
+const maxResultsPerMessage=100;
+let maxPKey = "";
+let maxPKeyUnsent = "";
+let allRows = [];
+
+let theInterval;
+
+
+try {
+    lastOutMaxPKey = maxPKey = fs.readFileSync("DBExtractor.maxPKey.dat", "utf8");
+    logger.log("debug", "MaxPKey Loaded: " + maxPKey);
+} catch (ex) {
+    if (ex.errno === -4058) {
+        logger.log("info", "MaxPKey-File not existing. Will be created.");
+    } else {
+        logger.log("error", "opening DBExtractor.maxPKey.dat :" + ex);
+    }
+}
+
+sql.connect(config.sqlConfig, err => {
+    if (err) {
+        logger.log("error", "Connecting to DB: " + err);
+    }
+    const sqlRequest = new sql.Request();
+    sqlRequest.stream = true;
+
+	function runNextPoll(){
+		sqlRequest.query("select top " + maxResultsPerMessage + " * from rpslog where pkey > '" + maxPKey + "' order by pkey");
+	}
+	
+    runNextPoll();
+
+    sqlRequest.on('row', row => {
+        outData = { sourceTag: config.sourceTag, processingType: config.processingType, data: row };
+        process.stdout.write("#");
+        request.json = true;
+
+        logger.log('debug', "adding row to array:" + JSON.stringify(outData));
+        allRows.push(outData);
+
+        if (row.PKey > maxPKeyUnsent) {
+            maxPKeyUnsent = row.PKey;
+            logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent);
+        }
+    });
+
+    sqlRequest.on('done', result => {
+		clearInterval(theInterval);
+        if (allRows.length>0){
+            request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
+                // count up the maxPKey only when the request was successful...
+                maxPKey=maxPKeyUnsent;
+                fs.writeFileSync("DBExtractor.maxPKey.dat", maxPKey);
+            }).catch(err=>{
+                logger.log("error","Sending Request to Service: " + err)
+            }).finally(_=>{
+				runNextPoll();
+			});
+            process.stdout.write(">" + allRows.length + ">");
+		} else {
+			theInterval = setInterval(runNextPoll, config.pollIntervalIdle);
+			process.stdout.write(".");
+        }
+        allRows=[];
+
+    });
+})

DBExtractor/DBExtractor.maxPKey.dat

@@ -0,0 +1 @@
+0010000000a87kj9