joe/logAggregator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lots of re-structuring and handling of different data types

Browse Source
This commit is contained in:
Joe
2019-01-18 15:29:21 -06:00
parent e1007f8ed9
commit b0d5fca2a4
8 changed files with 51 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
DBExtractor/DBExtractor.config.json Normal file
View File

@@ -0,0 +1,14 @@
{
"dbDriver":"mssql", "dbDriverComment":"either mssql (user/password) or mssql/msnodesqlv8 (trustedConnection)",
"processingType": "casrpslog",
"sourceTag": "CCERpsLog",
"serverURI": "http://localhost:2222",
"pollIntervalIdle": 10000,
"sqlConfig": {
"user": "casdbuser",
"password": "4KcLao2016!",
"server": "ustww2063\\CCE",
"database": "CAS110",
"port": 1433
}
}

2
casRpsLogExtractor/casRpsLogExtractor.config.json.example → DBExtractor/DBExtractor.config.json.example
View File

@@ -1,4 +1,6 @@
{
"dbDriver":"mssql", "dbDriverComment":"either mssql (user/password) or mssql/msnodesqlv8 (trustedConnection)",
"processingType": "casrpslog",
"sourceTag": "CCERpsLog",
"serverURI": "http://localhost:2222",
"pollIntervalIdle": 10000,

23
casRpsLogExtractor/casRpsLogExtractor.js → DBExtractor/DBExtractor.js
View File

@@ -1,7 +1,8 @@
const sql = require("mssql/msnodesqlv8");
//const sql = require("mssql");
const request = require("request-promise");
const fs = require("fs");
const config = JSON.parse(fs.readFileSync("DBExtractor.config.json", "utf8"));
const sql = require(config.dbDriver);
const request = require("request-promise");
const winston = require("winston");
const logger = winston.createLogger({
@@ -14,12 +15,12 @@ const logger = winston.createLogger({
),
transports: [
new winston.transports.Console(),
new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }),
new winston.transports.File({ filename: './log/DBExtractorError.log', level: 'error' }),
new winston.transports.File({ filename: './log/Combined.log' })
]
});
console.log(`casRpsLogExtractor Copyright (C) 2019 j.tretter@gmail.com
console.log(`DBExtractor Copyright (C) 2019 j.tretter@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -34,12 +35,10 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.`);
logger.log("info", `casRpsLogExtractor v0.1 Starting`);
logger.log("info", `DBExtractor v0.1 Starting`);
const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8"));
logger.log("debug", "Config" + JSON.stringify(config))
const processingType = "casrpslog";
const maxResultsPerMessage=100;
let maxPKey = "";
let maxPKeyUnsent = "";
@@ -49,13 +48,13 @@ let theInterval;
try {
lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
lastOutMaxPKey = maxPKey = fs.readFileSync("DBExtractor.maxPKey.dat", "utf8");
logger.log("debug", "MaxPKey Loaded: " + maxPKey);
} catch (ex) {
if (ex.errno === -4058) {
logger.log("info", "MaxPKey-File not existing. Will be created.");
} else {
logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex);
logger.log("error", "opening DBExtractor.maxPKey.dat :" + ex);
}
}
@@ -73,7 +72,7 @@ sql.connect(config.sqlConfig, err => {
runNextPoll();
sqlRequest.on('row', row => {
outData = { sourceTag: config.sourceTag, processingType: processingType, data: row };
outData = { sourceTag: config.sourceTag, processingType: config.processingType, data: row };
process.stdout.write("#");
request.json = true;
@@ -92,7 +91,7 @@ sql.connect(config.sqlConfig, err => {
request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
// count up the maxPKey only when the request was successful...
maxPKey=maxPKeyUnsent;
fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
fs.writeFileSync("DBExtractor.maxPKey.dat", maxPKey);
}).catch(err=>{
logger.log("error","Sending Request to Service: " + err)
}).finally(_=>{

1
DBExtractor/DBExtractor.maxPKey.dat Normal file
View File

@@ -0,0 +1 @@
0010000000a87kj9

27
RuleEngine.js
View File

@@ -107,26 +107,39 @@ exports.RuleEngine = class RuleEngine {
logger.log("debug", "Logic FLAT start - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
let isKnownComp = false;
let retVal = false;
let dataValue;
let ruleValue;
let sDataType = theWhiteFilterRule.dataType || "string";
if (sDataType ==="number") {
dataValue = Number(inputData[theWhiteFilterRule.Field]);
ruleValue = Number(theWhiteFilterRule.Value);
} else {
dataValue = inputData[theWhiteFilterRule.Field];
ruleValue = theWhiteFilterRule.Value;
}
try {
if (theWhiteFilterRule.Comp === "===") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {retVal = true;}
if (dataValue === ruleValue) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "<") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] < theWhiteFilterRule.Value) {retVal = true;}
if (dataValue < ruleValue) {retVal = true;}
} else if (theWhiteFilterRule.Comp === ">") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] > theWhiteFilterRule.Value) {retVal = true;}
if (dataValue > ruleValue) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "<=") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] <= theWhiteFilterRule.Value) {retVal = true;}
if (dataValue <= ruleValue) {retVal = true;}
} else if (theWhiteFilterRule.Comp === ">=") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] >= theWhiteFilterRule.Value) {retVal = true;}
if (dataValue >= ruleValue) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "~") {
isKnownComp = true;
logger.log("debug", "RegexFilter: " + theWhiteFilterRule.Value + " || " + ""+inputData[theWhiteFilterRule.Field].replace(/\r/g, "\\r").replace(/\n/g, "\\n"));
retVal = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
logger.log("debug", "RegexFilter: " + ruleValue + " || " + ""+dataValue.replace(/\r/g, "\\r").replace(/\n/g, "\\n"));
retVal = this.reTest(ruleValue, dataValue);
}
} catch (ex) {
logger.log("error", "LogicFlat-Error " + ex + " WhiteFilterRule: " + JSON.stringify(theWhiteFilterRule) + " InputData " + JSON.stringify(inputData));

5
diskUsageMonitor/diskUsageMonitor.js
View File

@@ -45,20 +45,15 @@ function cb(){
theMessage.processingType=processingType;
theMessage.sourceTag=config.sourceTag;
theMessage.timestamp=new Date();
theDiskLimit.isWarn=false;
disk.check(theDiskLimit.path, function(err, info) {
if (err) {
logger.log("error","Error reading disk Information " + err);
theDiskLimit.error = err;
theDiskLimit.isWarn = true;
} else {
let availGB=info.available/1024/1024;
let totalGB=info.total/1024/1024;
theDiskLimit.availGB=availGB;
theDiskLimit.totalGB=totalGB;
if (availGB<=theDiskLimit.minFree) {
theDiskLimit.isWarn=true;
}
}
theMessage.data=theDiskLimit;
logger.log("debug","Sending Message: " + JSON.stringify(theMessage));

BIN
log/.issues.log.un~
View File

Binary file not shown.

2
rules/diskusage.rule.json
View File

@@ -1,5 +1,5 @@
{ "diskusage":
{ "rules":[
{ "Logic":"-", "Field":"isWarn", "Comp":"!==", "Value":true}
{ "Logic":"-", "dataType":"number", "Field":"availGB", "Comp":">", "Value":100}
]}
}