joe/logAggregator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handling Requests multimessage

Browse Source
This commit is contained in:
Joe
2019-01-07 07:51:40 -06:00
parent bdf24867f8
commit e29b4b9e86
3 changed files with 40 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
casRpsLogExtractor/casRpsLogExtractor.js
View File

@@ -1,5 +1,5 @@
const sql = require("mssql/msnodesqlv8"); //const sql = require("mssql/msnodesqlv8");
//const sql = require("mssql"); const sql = require("mssql");
const request = require("request-promise"); const request = require("request-promise");
const fs = require("fs"); const fs = require("fs");
const winston = require("winston"); const winston = require("winston");
@@ -25,7 +25,9 @@ const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf
logger.log("debug", "Config" + JSON.stringify(config)) logger.log("debug", "Config" + JSON.stringify(config))
const processingType = "casrpslog"; const processingType = "casrpslog";
let maxPKey = ""; let maxPKey = "";
let maxPKeyUnsent = "";
let lastOutMaxPKey = ""; let lastOutMaxPKey = "";
let allRows = [];
try { try {
lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8"); lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
logger.log("debug", "MaxPKey Loaded: " + maxPKey); logger.log("debug", "MaxPKey Loaded: " + maxPKey);
@@ -53,20 +55,28 @@ sql.connect(config.sqlConfig, err => {
process.stdout.write("#"); process.stdout.write("#");
request.json = true; request.json = true;
logger.log('debug', "sending data:" + JSON.stringify(outData)); logger.log('debug', "adding row to array:" + JSON.stringify(outData));
request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(outData), json: true }).then(_ => { allRows.push(outData);
// count up the maxPKey only when the request was successful...
if (row.PKey > maxPKey) { if (row.PKey > maxPKeyUnsent) {
maxPKey = row.PKey; maxPKeyUnsent = row.PKey;
fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey); logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent);
logger.log("debug", "MaxPKey Written: " + maxPKey); }
}
}).catch(err=>{
logger.log("error","Sending Request to Service: " + err)
});
}); });
sqlRequest.on('done', result => { sqlRequest.on('done', result => {
if (allRows.length>0){
request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
// count up the maxPKey only when the request was successful...
maxPKey=maxPKeyUnsent;
fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
}).catch(err=>{
logger.log("error","Sending Request to Service: " + err)
});
process.stdout.write(">" + allRows.length + ">");
allRows=[];
}
if (lastOutMaxPKey !== maxPKey) { if (lastOutMaxPKey !== maxPKey) {
process.stdout.write("[" + maxPKey + "]\n"); process.stdout.write("[" + maxPKey + "]\n");
lastOutMaxPKey = maxPKey; lastOutMaxPKey = maxPKey;

1
casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat Normal file
View File

@@ -0,0 +1 @@
0010000000a87kj9

28
logAggregator.js
View File

@@ -27,23 +27,27 @@ const server = micro(async (req, res) => {
const js = await json(req); const js = await json(req);
try { try {
inputLine = JSON.parse(js); allInputLines = JSON.parse(js);
send(res, 200, "OK");
} catch (ex) { } catch (ex) {
logger.log("error", "Input not in JSON format:" + line); logger.log("error", "Input not in JSON format:" + js);
send(res, 401, "Input not in JSON Format");
} }
let isWhiteListed = ruleEngine.processData(inputLine); for(i=0;i<allInputLines.length;i++){
let miniInput = jsonminify(JSON.stringify(inputLine)); let theInputLine=allInputLines[i];
let now = Date(); let isWhiteListed = ruleEngine.processData(theInputLine);
if (isWhiteListed) { let miniInput = jsonminify(JSON.stringify(theInputLine));
logger.log("debug", "Not Whitelisted: " + miniInput); let now = Date();
fs.appendFileSync("./log/skipped.log", miniInput + "\n"); if (isWhiteListed) {
} else { logger.log("debug", "Not Whitelisted: " + miniInput);
logger.log("info", "Not Whitelisted: " + miniInput); fs.appendFileSync("./log/skipped.log", miniInput + "\n");
fs.appendFileSync("./log/issues.log", miniInput + "\n"); } else {
logger.log("info", "Not Whitelisted: " + miniInput);
fs.appendFileSync("./log/issues.log", miniInput + "\n");
}
} }
send(res, 200, "OK");
}); });
server.listen(2222); server.listen(2222);