diff --git a/RuleEngine.js b/RuleEngine.js index cc5a108..bf47b20 100644 --- a/RuleEngine.js +++ b/RuleEngine.js @@ -16,14 +16,35 @@ const logger = winston.createLogger({ ] }); - exports.RuleEngine = class RuleEngine { - constructor(ruleSetFileMask) { - this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFileMask + '.rule.json', 'utf8')); + constructor(ruleSetFiles) { + logger.log("info", "Instanciated RulesEngine - RuleSetFiles: " + ruleSetFiles); + this.ruleSet = {}; + if (ruleSetFiles === "*") { + fs.readdirSync("./rules/").forEach(file => { + if (this.reTest("/.*\.rule\.json$/", file)) { + logger.log("info", "Loading ruleset file " + file); + let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file, 'utf8')); + logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]); + this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]] + } + }); + } else if (ruleSetFiles.indexOf(",") > 1) { + ruleSetFiles.split(",").forEach(file => { + logger.log("info", "Loading ruleset file " + file); + let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file + '.rule.json', 'utf8')); + logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]); + this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]] + }); + } else { + logger.log("info", "Loading ruleset file " + ruleSetFiles); + this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFiles + '.rule.json', 'utf8')); + logger.log("info", "Adding Ruleset " + Object.keys(this.ruleSet)[0]); + } } processData(inputLine) { let inputData = inputLine.data; - logger.log("debug", "ProcessingType " + inputLine.processingType + " RuleSet: " + this.ruleSet); + logger.log("debug", "ProcessingType " + inputLine.processingType); let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules; if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); } logger.log("debug", "No whitelisting matched; returning false"); @@ -46,9 +67,9 @@ exports.RuleEngine = class RuleEngine { retVal = false; }; } else if (theWhiteFilterRule.Logic === "&") { - if (!this.processLogicAnd(theWhiteFilterRule.rules, inputData) && retVal === true) { + if (!this.processLogicAnd(theWhiteFilterRule.rules, inputData) && retVal === true) { logger.log("debug", " Logic AND - AND - Changing Result to false"); - retVal = false; + retVal = false; }; } }; @@ -59,45 +80,61 @@ exports.RuleEngine = class RuleEngine { logger.log("debug", "Logic OR - start"); for (let i = 0; i < allWhiteFilterRules.length; i++) { let theWhiteFilterRule = allWhiteFilterRules[i]; - logger.log("debug", " Logic OR - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule)); + logger.log("debug", " Logic OR - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule)); if (theWhiteFilterRule.Logic === "-") { if (this.processLogicFlat(theWhiteFilterRule, inputData)) { logger.log("debug", " Logic OR - FLAT - RETURNING OUT"); - return (true); - }; + return (true); + }; } else if (theWhiteFilterRule.Logic === "|") { - if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) { + if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) { logger.log("debug", " Logic OR - OR - RETURNING OUT"); - return (true) + return (true) }; } else if (theWhiteFilterRule.Logic === "&") { - if (this.processLogicAnd(theWhiteFilterRule.rules, inputData)) { + if (this.processLogicAnd(theWhiteFilterRule.rules, inputData)) { logger.log("debug", " Logic OR - AND - RETURNING OUT"); - return (true) + return (true) }; } }; logger.log("debug", "Logic OR - end"); - return(false); + return (false); } processLogicFlat(theWhiteFilterRule, inputData) { - logger.log("debug", "Logic FLAT start - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule)); + logger.log("debug", "Logic FLAT start - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule)); + let isKnownComp = false; + let retVal = false; if (theWhiteFilterRule.Comp === "===") { - if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) { - logger.log("debug", "Logic FLAT - Exiting out"); - return (true); - } + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {retVal = true;} + } else if (theWhiteFilterRule.Comp === "<") { + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] < theWhiteFilterRule.Value) {retVal = true;} + } else if (theWhiteFilterRule.Comp === ">") { + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] > theWhiteFilterRule.Value) {retVal = true;} + } else if (theWhiteFilterRule.Comp === "<=") { + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] <= theWhiteFilterRule.Value) {retVal = true;} + } else if (theWhiteFilterRule.Comp === ">=") { + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] >= theWhiteFilterRule.Value) {retVal = true;} } else if (theWhiteFilterRule.Comp === "~") { - logger.log("debug","RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field]); + isKnownComp = true; + logger.log("debug", "RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field].replace(/\r/g, "\\r").replace(/\n/g, "\\n")); + retVal = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]); + } - let result = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]); - - if (result) { - logger.log("debug", "Logic FLAT - Exiting out"); - return (true); - } + if (!isKnownComp) { + logger.log("error", "The configured Comp '" + theWhiteFilterRule.Comp + "' is unknown.") + } + + if (retVal === true) { + logger.log("debug", "Logic FLAT - Exiting out"); } logger.log("debug", "Logic FLAT - end"); + return (retVal); } reTest(regex, data) { diff --git a/casRpsLogExtractor/casRpsLogExtractor.js b/casRpsLogExtractor/casRpsLogExtractor.js index f97dc2d..cf0f971 100644 --- a/casRpsLogExtractor/casRpsLogExtractor.js +++ b/casRpsLogExtractor/casRpsLogExtractor.js @@ -60,6 +60,8 @@ sql.connect(config.sqlConfig, err => { fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey); logger.log("debug", "MaxPKey Written: " + maxPKey); } + }).catch(err=>{ + logger.log("error","Sending Request to Service: " + err) }); }); diff --git a/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat index 079bd55..df706b3 100644 --- a/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat +++ b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat @@ -1 +1 @@ -0010000000a87kiz \ No newline at end of file +0010000000a87kj9 \ No newline at end of file diff --git a/logAggregator.js b/logAggregator.js index 87c8c5f..71a8f4e 100644 --- a/logAggregator.js +++ b/logAggregator.js @@ -21,27 +21,27 @@ const logger = winston.createLogger({ }); -const ruleEngine = new RuleEngine("casrpslog"); -logger.log("info","Service Starting."); +const ruleEngine = new RuleEngine("*"); +logger.log("info", "Service Starting."); const server = micro(async (req, res) => { const js = await json(req); try { inputLine = JSON.parse(js); } catch (ex) { - logger.log("error","Input not in JSON format:" + line); + logger.log("error", "Input not in JSON format:" + line); } let isWhiteListed = ruleEngine.processData(inputLine); - let miniInput=jsonminify(JSON.stringify(inputLine)) ; + let miniInput = jsonminify(JSON.stringify(inputLine)); let now = Date(); if (isWhiteListed) { - logger.log("debug","Not Whitelisted: " + miniInput); - fs.appendFileSync("./log/skipped.log",miniInput + "\n"); + logger.log("debug", "Not Whitelisted: " + miniInput); + fs.appendFileSync("./log/skipped.log", miniInput + "\n"); } else { - logger.log("info","Not Whitelisted: " + miniInput); - fs.appendFileSync("./log/issues.log",miniInput + "\n"); - } + logger.log("info", "Not Whitelisted: " + miniInput); + fs.appendFileSync("./log/issues.log", miniInput + "\n"); + } send(res, 200, "OK"); }); diff --git a/rules/casroslogfull.rule.json b/rules/casroslogfull.rule.json deleted file mode 100644 index 6d5aecd..0000000 --- a/rules/casroslogfull.rule.json +++ /dev/null @@ -1,26 +0,0 @@ -{ "casrpslog":{ - "rules":[ - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, - {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"}, - {"Logic":"&", "rules":[ - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"}, - {"Logic":"|", "rules":[ - {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"}, - {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"} - ]} - ] - }, - {"Logic":"&", "rules":[ - {"Logic":"|", "rules":[ - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"}, - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"} - ]}, - {"Logic":"|", "rules":[ - {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."}, - {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."}, - {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"} - ]} - ]} - ] - } - } \ No newline at end of file diff --git a/rules/casrpslog.rule.json b/rules/casrpslog.rule.json index bb4a475..f6d2c36 100644 --- a/rules/casrpslog.rule.json +++ b/rules/casrpslog.rule.json @@ -1,16 +1,16 @@ { "casrpslog":{ "rules":[ - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, - {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"}, - {"Logic":"&", "rules":[ - {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"}, + { "_comment":"Application Validation Errors", + "Logic":"&", "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CAS 11.0"}, {"Logic":"|", "rules":[ {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"}, - {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"} + {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/^The promotion or tactic is not allowed to extend past the last day of the year.*/"} ]} ] }, - {"Logic":"&", "rules":[ + {"_comment":"Worker Process info messages", + "Logic":"&", "rules":[ {"Logic":"|", "rules":[ {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"}, {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"},