From e45e5258955f8bd578d577a3aa90301b494c1eed Mon Sep 17 00:00:00 2001 From: Tretter Date: Fri, 4 Jan 2019 11:06:37 -0600 Subject: [PATCH] All kinds of great changes --- RuleEngine.js | 108 ++++++++++++ casRpsLogExtractor.js | 43 ----- .../casRpsLogExtractor.config.json | 12 ++ casRpsLogExtractor/casRpsLogExtractor.js | 74 ++++++++ .../casRpsLogExtractor.maxPKey.dat | 1 + index.js | 35 ---- logAggregator.js | 49 ++++++ package-lock.json | 161 ++++++++++++++++++ package.json | 9 +- rules/casroslogfull.rule.json | 26 +++ rules/casrpslog.rule.json | 27 +++ 11 files changed, 464 insertions(+), 81 deletions(-) create mode 100644 RuleEngine.js delete mode 100644 casRpsLogExtractor.js create mode 100644 casRpsLogExtractor/casRpsLogExtractor.config.json create mode 100644 casRpsLogExtractor/casRpsLogExtractor.js create mode 100644 casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat delete mode 100644 index.js create mode 100644 logAggregator.js create mode 100644 rules/casroslogfull.rule.json create mode 100644 rules/casrpslog.rule.json diff --git a/RuleEngine.js b/RuleEngine.js new file mode 100644 index 0000000..cc5a108 --- /dev/null +++ b/RuleEngine.js @@ -0,0 +1,108 @@ +const fs = require('fs'); +const winston = require("winston"); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf(info => { + return `${info.timestamp} ${info.level}: ${info.message}`; + }) + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: './log/RuleEngineError.log', level: 'error' }), + new winston.transports.File({ filename: './log/Combined.log' }) + ] +}); + + +exports.RuleEngine = class RuleEngine { + constructor(ruleSetFileMask) { + this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFileMask + '.rule.json', 'utf8')); + } + processData(inputLine) { + let inputData = inputLine.data; + logger.log("debug", "ProcessingType " + inputLine.processingType + " RuleSet: " + this.ruleSet); + let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules; + if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); } + logger.log("debug", "No whitelisting matched; returning false"); + return (false); + } + processLogicAnd(allWhiteFilterRules, inputData) { + let retVal = true; + logger.log("debug", "Logic AND - start"); + for (let i = 0; i < allWhiteFilterRules.length; i++) { + let theWhiteFilterRule = allWhiteFilterRules[i]; + logger.log("debug", " Logic AND - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule)); + if (theWhiteFilterRule.Logic === "-") { + if (!this.processLogicFlat(theWhiteFilterRule, inputData) && retVal === true) { + logger.log("debug", " Logic AND - FLAT - Changing Result to false"); + retVal = false; + }; + } else if (theWhiteFilterRule.Logic === "|") { + if (!this.processLogicOr(theWhiteFilterRule.rules, inputData) && retVal === true) { + logger.log("debug", " Logic AND - OR - Changing Result to false"); + retVal = false; + }; + } else if (theWhiteFilterRule.Logic === "&") { + if (!this.processLogicAnd(theWhiteFilterRule.rules, inputData) && retVal === true) { + logger.log("debug", " Logic AND - AND - Changing Result to false"); + retVal = false; + }; + } + }; + logger.log("debug", "Logic AND - end - result: " + retVal); + return (retVal); + } + processLogicOr(allWhiteFilterRules, inputData) { + logger.log("debug", "Logic OR - start"); + for (let i = 0; i < allWhiteFilterRules.length; i++) { + let theWhiteFilterRule = allWhiteFilterRules[i]; + logger.log("debug", " Logic OR - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule)); + if (theWhiteFilterRule.Logic === "-") { + if (this.processLogicFlat(theWhiteFilterRule, inputData)) { + logger.log("debug", " Logic OR - FLAT - RETURNING OUT"); + return (true); + }; + } else if (theWhiteFilterRule.Logic === "|") { + if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) { + logger.log("debug", " Logic OR - OR - RETURNING OUT"); + return (true) + }; + } else if (theWhiteFilterRule.Logic === "&") { + if (this.processLogicAnd(theWhiteFilterRule.rules, inputData)) { + logger.log("debug", " Logic OR - AND - RETURNING OUT"); + return (true) + }; + } + }; + logger.log("debug", "Logic OR - end"); + return(false); + } + processLogicFlat(theWhiteFilterRule, inputData) { + logger.log("debug", "Logic FLAT start - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule)); + if (theWhiteFilterRule.Comp === "===") { + if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) { + logger.log("debug", "Logic FLAT - Exiting out"); + return (true); + } + } else if (theWhiteFilterRule.Comp === "~") { + logger.log("debug","RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field]); + + let result = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]); + + if (result) { + logger.log("debug", "Logic FLAT - Exiting out"); + return (true); + } + } + logger.log("debug", "Logic FLAT - end"); + } + + reTest(regex, data) { + let match = regex.match(new RegExp('^/(.*?)/([giumy]*)$')); + regex = new RegExp(match[1], match[2]); + return regex.test(data); + } +} diff --git a/casRpsLogExtractor.js b/casRpsLogExtractor.js deleted file mode 100644 index 90adad0..0000000 --- a/casRpsLogExtractor.js +++ /dev/null @@ -1,43 +0,0 @@ -let sql=require("mssql"); -let request=require("request-promise"); - -//sql.trustedConnection = true; -let sqlConfig = { - user: 'casdbuser', - password:'4KcLao2016!', - server: 'ustww2063\\CCE', - database: 'CAS110', - port: 1433 -} - -let sourceTag="CCERpsLog"; -let processingType="casrpslog"; -let maxPKey=""; - -sql.connect(sqlConfig,err => { - console.log("JOE" , err); - const sqlRequest = new sql.Request(); - sqlRequest.stream = true; - - - setInterval(_=>{ - sqlRequest.query('select top 1 * from rpslog where pkey > \'' + maxPKey + '\' order by pkey'); - },500) - - sqlRequest.on('row',row=>{ - outData={sourceTag:sourceTag,processingType:processingType,data:row}; - console.log(outData); - request.json = true; - request({method:'POST',uri:'http://localhost:2222',body:JSON.stringify(outData),json:true}).then(_=>{ - // count up the maxPKey only when the request was successful... - if (row.PKey > maxPKey) { - maxPKey=row.PKey; - } - }); - - }); - - sqlRequest.on('done', result=>{ - console.log("Done"); - }); -}) diff --git a/casRpsLogExtractor/casRpsLogExtractor.config.json b/casRpsLogExtractor/casRpsLogExtractor.config.json new file mode 100644 index 0000000..e101958 --- /dev/null +++ b/casRpsLogExtractor/casRpsLogExtractor.config.json @@ -0,0 +1,12 @@ +{ + "sourceTag": "CCERpsLog", + "serverURI": "http://localhost:2222", + "pollInterval": 10000, + "sqlConfig": { + "user": "casdbuser", + "password": "4KcLao2016!", + "server": "ustww2063\\CCE", + "database": "CAS110", + "port": 1433 + } +} \ No newline at end of file diff --git a/casRpsLogExtractor/casRpsLogExtractor.js b/casRpsLogExtractor/casRpsLogExtractor.js new file mode 100644 index 0000000..f97dc2d --- /dev/null +++ b/casRpsLogExtractor/casRpsLogExtractor.js @@ -0,0 +1,74 @@ +const sql = require("mssql"); +const request = require("request-promise"); +const fs = require("fs"); +const winston = require("winston"); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf(info => { + return `${info.timestamp} ${info.level}: ${info.message}`; + }) + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }), + new winston.transports.File({ filename: './log/Combined.log' }) + ] +}); + +logger.log("info", "Starting casRpsLogExtractor"); +const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8")); + +logger.log("debug", "Config" + JSON.stringify(config)) +const processingType = "casrpslog"; +let maxPKey = ""; +let lastOutMaxPKey = ""; +try { + lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8"); + logger.log("debug", "MaxPKey Loaded: " + maxPKey); +} catch (ex) { + if (ex.errno === -4058) { + logger.log("info", "MaxPKey-File not existing. Will be created."); + } else { + logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex); + } +} + +sql.connect(config.sqlConfig, err => { + if (err) { + logger.log("error", "Connecting to DB: " + err); + } + const sqlRequest = new sql.Request(); + sqlRequest.stream = true; + + setInterval(_ => { + sqlRequest.query("select top 100 * from rpslog where pkey > '" + maxPKey + "' order by pkey"); + }, config.pollInterval) + + sqlRequest.on('row', row => { + outData = { sourceTag: config.sourceTag, processingType: processingType, data: row }; + process.stdout.write("#"); + request.json = true; + + logger.log('debug', "sending data:" + JSON.stringify(outData)); + request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(outData), json: true }).then(_ => { + // count up the maxPKey only when the request was successful... + if (row.PKey > maxPKey) { + maxPKey = row.PKey; + fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey); + logger.log("debug", "MaxPKey Written: " + maxPKey); + } + }); + }); + + sqlRequest.on('done', result => { + if (lastOutMaxPKey !== maxPKey) { + process.stdout.write("[" + maxPKey + "]\n"); + lastOutMaxPKey = maxPKey; + } else { + process.stdout.write("."); + } + }); +}) diff --git a/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat new file mode 100644 index 0000000..079bd55 --- /dev/null +++ b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat @@ -0,0 +1 @@ +0010000000a87kiz \ No newline at end of file diff --git a/index.js b/index.js deleted file mode 100644 index 4bcf8bf..0000000 --- a/index.js +++ /dev/null @@ -1,35 +0,0 @@ -const micro = require('micro') -const {json,send} = require('micro') - -const reTest= (regex,data)=>{ - let match = regex.match(new RegExp('^/(.*?)/([giumy]*)$')); - regex = new RegExp(match[1], match[2]); - return regex.test(data); -} - -// SourceTag -// ProcessingType - -const server = micro(async (req,res) => { - //console.log(req); - const js = await json(req); - //console.log("Received Data",js); - - try { - inputLine = JSON.parse(js); - } catch (ex) { - console.error("Input not in JSON format:" + line); - } - - if (inputLine.processingType==="casrpslog") { - if (inputLine.data.Severity > 1 ) { - console.log("Issue ",inputLine.data.LogText); - } - } - //console.log("line: " + reTest("/error/i",inputLine)); - - - send(res,200,"OK"); -}); - -server.listen(2222); \ No newline at end of file diff --git a/logAggregator.js b/logAggregator.js new file mode 100644 index 0000000..87c8c5f --- /dev/null +++ b/logAggregator.js @@ -0,0 +1,49 @@ +const micro = require('micro'); +const { json, send } = require('micro'); +const { RuleEngine } = require('./RuleEngine'); +const fs = require('fs'); +const jsonminify = require('jsonminify'); +const winston = require("winston"); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf(info => { + return `${info.timestamp} ${info.level}: ${info.message}`; + }) + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: './log/LogAggregatorError.log', level: 'error' }), + new winston.transports.File({ filename: './log/Combined.log' }) + ] +}); + + +const ruleEngine = new RuleEngine("casrpslog"); +logger.log("info","Service Starting."); +const server = micro(async (req, res) => { + const js = await json(req); + + try { + inputLine = JSON.parse(js); + } catch (ex) { + logger.log("error","Input not in JSON format:" + line); + } + + let isWhiteListed = ruleEngine.processData(inputLine); + let miniInput=jsonminify(JSON.stringify(inputLine)) ; + let now = Date(); + if (isWhiteListed) { + logger.log("debug","Not Whitelisted: " + miniInput); + fs.appendFileSync("./log/skipped.log",miniInput + "\n"); + } else { + logger.log("info","Not Whitelisted: " + miniInput); + fs.appendFileSync("./log/issues.log",miniInput + "\n"); + } + + send(res, 200, "OK"); +}); + +server.listen(2222); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8e9a0cb..ef5ac24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,14 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", + "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "requires": { + "lodash": "^4.17.10" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -112,6 +120,15 @@ "supports-color": "^5.3.0" } }, + "color": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", + "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -125,6 +142,34 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "color-string": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz", + "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "colornames": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz", + "integrity": "sha1-+IiQMGhcfE/54qVZ9Qd+t2qBb5Y=" + }, + "colors": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.3.3.tgz", + "integrity": "sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg==" + }, + "colorspace": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.1.tgz", + "integrity": "sha512-pI3btWyiuz7Ken0BWh9Elzsmv2bM9AhA7psXib4anUXy/orfZ/E0MbQwhSOG/9L8hLlalqrU0UhOuqxW1YjmVw==", + "requires": { + "color": "3.0.x", + "text-hex": "1.0.x" + } + }, "combined-stream": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", @@ -174,6 +219,16 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, + "diagnostics": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", + "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==", + "requires": { + "colorspace": "1.1.x", + "enabled": "1.0.x", + "kuler": "1.0.x" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -183,6 +238,19 @@ "safer-buffer": "^2.1.0" } }, + "enabled": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", + "integrity": "sha1-ll9lE9LC0cX0ZStkouM5ZGf8L5M=", + "requires": { + "env-variable": "0.0.x" + } + }, + "env-variable": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", + "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -208,6 +276,16 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, + "fast-safe-stringify": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz", + "integrity": "sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg==" + }, + "fecha": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", + "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -296,6 +374,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -336,6 +419,11 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsonminify": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/jsonminify/-/jsonminify-0.4.1.tgz", + "integrity": "sha1-gF2vuzk5UYjO6atYLIHvlZ1+cQw=" + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -347,11 +435,31 @@ "verror": "1.10.0" } }, + "kuler": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz", + "integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==", + "requires": { + "colornames": "^1.1.1" + } + }, "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, + "logform": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", + "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", + "requires": { + "colors": "^1.2.1", + "fast-safe-stringify": "^2.0.4", + "fecha": "^2.3.3", + "ms": "^2.1.1", + "triple-beam": "^1.2.0" + } + }, "micro": { "version": "9.3.3", "resolved": "https://registry.npmjs.org/micro/-/micro-9.3.3.tgz", @@ -407,6 +515,11 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "one-time": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", + "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4=" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -535,6 +648,14 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "requires": { + "is-arrayish": "^0.3.1" + } + }, "sprintf-js": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", @@ -556,6 +677,11 @@ "tweetnacl": "~0.14.0" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -598,6 +724,11 @@ "sprintf-js": "^1.1.1" } }, + "text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", @@ -614,6 +745,11 @@ } } }, + "triple-beam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", + "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -659,6 +795,31 @@ "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } + }, + "winston": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.1.0.tgz", + "integrity": "sha512-FsQfEE+8YIEeuZEYhHDk5cILo1HOcWkGwvoidLrDgPog0r4bser1lEIOco2dN9zpDJ1M88hfDgZvxe5z4xNcwg==", + "requires": { + "async": "^2.6.0", + "diagnostics": "^1.1.1", + "is-stream": "^1.1.0", + "logform": "^1.9.1", + "one-time": "0.0.4", + "readable-stream": "^2.3.6", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.2.0" + } + }, + "winston-transport": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz", + "integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==", + "requires": { + "readable-stream": "^2.3.6", + "triple-beam": "^1.2.0" + } } } } diff --git a/package.json b/package.json index 0115fdd..43a4a8c 100644 --- a/package.json +++ b/package.json @@ -2,18 +2,21 @@ "name": "logaggregator", "version": "1.0.0", "description": "", - "main": "index.js", + "main": "logAggregator.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node logAggregator.js" }, "author": "", "license": "ISC", "dependencies": { + "jsonminify": "^0.4.1", "micro": "^9.3.3", "msnodesqlv8": "^0.6.11", "mssql": "^4.3.0", "readline": "^1.3.0", "request": "^2.88.0", - "request-promise": "^4.2.2" + "request-promise": "^4.2.2", + "winston": "^3.1.0" } } diff --git a/rules/casroslogfull.rule.json b/rules/casroslogfull.rule.json new file mode 100644 index 0000000..6d5aecd --- /dev/null +++ b/rules/casroslogfull.rule.json @@ -0,0 +1,26 @@ +{ "casrpslog":{ + "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, + {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"}, + {"Logic":"&", "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"}, + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"}, + {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"} + ]} + ] + }, + {"Logic":"&", "rules":[ + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"}, + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"} + ]}, + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."}, + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."}, + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"} + ]} + ]} + ] + } + } \ No newline at end of file diff --git a/rules/casrpslog.rule.json b/rules/casrpslog.rule.json new file mode 100644 index 0000000..bb4a475 --- /dev/null +++ b/rules/casrpslog.rule.json @@ -0,0 +1,27 @@ +{ "casrpslog":{ + "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, + {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"}, + {"Logic":"&", "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"}, + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"}, + {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"} + ]} + ] + }, + {"Logic":"&", "rules":[ + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"}, + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"}, + {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationInboundWorker"} + ]}, + {"Logic":"|", "rules":[ + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."}, + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."}, + {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"} + ]} + ]} + ] + } +} \ No newline at end of file