All kinds of great changes

2019-01-04 11:06:37 -06:00
parent 27207dcab0
commit e45e525895
11 changed files with 464 additions and 81 deletions
--- a/RuleEngine.js
+++ b/RuleEngine.js
@@ -0,0 +1,108 @@
+const fs = require('fs');
+const winston = require("winston");
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => {
+            return `${info.timestamp} ${info.level}: ${info.message}`;
+        })
+    ),
+    transports: [
+        new winston.transports.Console(),
+        new winston.transports.File({ filename: './log/RuleEngineError.log', level: 'error' }),
+        new winston.transports.File({ filename: './log/Combined.log' })
+    ]
+});
+
+
+exports.RuleEngine = class RuleEngine {
+    constructor(ruleSetFileMask) {
+        this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFileMask + '.rule.json', 'utf8'));
+    }
+    processData(inputLine) {
+        let inputData = inputLine.data;
+        logger.log("debug", "ProcessingType " + inputLine.processingType + " RuleSet: " + this.ruleSet);
+        let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules;
+        if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); }
+        logger.log("debug", "No whitelisting matched; returning false");
+        return (false);
+    }
+    processLogicAnd(allWhiteFilterRules, inputData) {
+        let retVal = true;
+        logger.log("debug", "Logic AND - start");
+        for (let i = 0; i < allWhiteFilterRules.length; i++) {
+            let theWhiteFilterRule = allWhiteFilterRules[i];
+            logger.log("debug", "  Logic AND - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
+            if (theWhiteFilterRule.Logic === "-") {
+                if (!this.processLogicFlat(theWhiteFilterRule, inputData) && retVal === true) {
+                    logger.log("debug", "    Logic AND - FLAT - Changing Result to false");
+                    retVal = false;
+                };
+            } else if (theWhiteFilterRule.Logic === "|") {
+                if (!this.processLogicOr(theWhiteFilterRule.rules, inputData) && retVal === true) {
+                    logger.log("debug", "    Logic AND - OR - Changing Result to false");
+                    retVal = false;
+                };
+            } else if (theWhiteFilterRule.Logic === "&") {
+                if (!this.processLogicAnd(theWhiteFilterRule.rules, inputData) && retVal === true) { 
+                    logger.log("debug", "    Logic AND - AND - Changing Result to false");
+                    retVal = false; 
+                };
+            }
+        };
+        logger.log("debug", "Logic AND - end - result: " + retVal);
+        return (retVal);
+    }
+    processLogicOr(allWhiteFilterRules, inputData) {
+        logger.log("debug", "Logic OR - start");
+        for (let i = 0; i < allWhiteFilterRules.length; i++) {
+            let theWhiteFilterRule = allWhiteFilterRules[i];
+            logger.log("debug", "  Logic OR - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
+            if (theWhiteFilterRule.Logic === "-") {
+                if (this.processLogicFlat(theWhiteFilterRule, inputData)) {
+                    logger.log("debug", "  Logic OR - FLAT - RETURNING OUT");
+                        return (true); 
+                    };
+            } else if (theWhiteFilterRule.Logic === "|") {
+                if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) { 
+                    logger.log("debug", "  Logic OR - OR - RETURNING OUT");
+                    return (true) 
+                };
+            } else if (theWhiteFilterRule.Logic === "&") {
+                if (this.processLogicAnd(theWhiteFilterRule.rules, inputData)) { 
+                    logger.log("debug", "  Logic OR - AND - RETURNING OUT");
+                    return (true) 
+                };
+            }
+        };
+        logger.log("debug", "Logic OR - end");
+        return(false);
+    }
+    processLogicFlat(theWhiteFilterRule, inputData) {
+        logger.log("debug", "Logic FLAT start - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
+        if (theWhiteFilterRule.Comp === "===") {
+            if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) { 
+                logger.log("debug", "Logic FLAT - Exiting out");
+                return (true); 
+            }
+        } else if (theWhiteFilterRule.Comp === "~") {
+            logger.log("debug","RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field]);
+
+            let result = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
+            
+            if (result) { 
+                logger.log("debug", "Logic FLAT - Exiting out");
+                return (true); 
+            }
+        }
+        logger.log("debug", "Logic FLAT - end");
+    }
+
+    reTest(regex, data) {
+        let match = regex.match(new RegExp('^/(.*?)/([giumy]*)$'));
+        regex = new RegExp(match[1], match[2]);
+        return regex.test(data);
+    }
+}
--- a/casRpsLogExtractor.js
+++ b/casRpsLogExtractor.js
@@ -1,43 +0,0 @@
-let sql=require("mssql");
-let request=require("request-promise");
-
-//sql.trustedConnection = true;
-let sqlConfig = {
-    user: 'casdbuser',
-    password:'4KcLao2016!',
-    server: 'ustww2063\\CCE',
-    database: 'CAS110',
-    port: 1433
-}
-
-let sourceTag="CCERpsLog";
-let processingType="casrpslog";
-let maxPKey="";
-
-sql.connect(sqlConfig,err => {
-    console.log("JOE" , err);
-    const sqlRequest = new sql.Request();
-    sqlRequest.stream = true;
-
-
-    setInterval(_=>{
-        sqlRequest.query('select top 1 * from rpslog where pkey > \'' + maxPKey + '\' order by pkey');
-    },500)
-
-    sqlRequest.on('row',row=>{
-        outData={sourceTag:sourceTag,processingType:processingType,data:row};
-        console.log(outData);
-        request.json = true;
-        request({method:'POST',uri:'http://localhost:2222',body:JSON.stringify(outData),json:true}).then(_=>{
-            // count up the maxPKey only when the request was successful...
-            if (row.PKey > maxPKey) {
-                maxPKey=row.PKey;
-            }
-        });
-
-    });
-
-    sqlRequest.on('done', result=>{
-        console.log("Done");
-    });
-})
--- a/casRpsLogExtractor/casRpsLogExtractor.config.json
+++ b/casRpsLogExtractor/casRpsLogExtractor.config.json
@@ -0,0 +1,12 @@
+{
+    "sourceTag": "CCERpsLog",
+    "serverURI": "http://localhost:2222",
+    "pollInterval": 10000,
+    "sqlConfig": {
+        "user": "casdbuser",
+        "password": "4KcLao2016!",
+        "server": "ustww2063\\CCE",
+        "database": "CAS110",
+        "port": 1433
+    }
+}
--- a/casRpsLogExtractor/casRpsLogExtractor.js
+++ b/casRpsLogExtractor/casRpsLogExtractor.js
@@ -0,0 +1,74 @@
+const sql = require("mssql");
+const request = require("request-promise");
+const fs = require("fs");
+const winston = require("winston");
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => {
+            return `${info.timestamp} ${info.level}: ${info.message}`;
+        })
+    ),
+    transports: [
+        new winston.transports.Console(),
+        new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }),
+        new winston.transports.File({ filename: './log/Combined.log' })
+    ]
+});
+
+logger.log("info", "Starting casRpsLogExtractor");
+const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8"));
+
+logger.log("debug", "Config" + JSON.stringify(config))
+const processingType = "casrpslog";
+let maxPKey = "";
+let lastOutMaxPKey = "";
+try {
+    lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
+    logger.log("debug", "MaxPKey Loaded: " + maxPKey);
+} catch (ex) {
+    if (ex.errno === -4058) {
+        logger.log("info", "MaxPKey-File not existing. Will be created.");
+    } else {
+        logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex);
+    }
+}
+
+sql.connect(config.sqlConfig, err => {
+    if (err) {
+        logger.log("error", "Connecting to DB: " + err);
+    }
+    const sqlRequest = new sql.Request();
+    sqlRequest.stream = true;
+
+    setInterval(_ => {
+        sqlRequest.query("select top 100 * from rpslog where pkey > '" + maxPKey + "' order by pkey");
+    }, config.pollInterval)
+
+    sqlRequest.on('row', row => {
+        outData = { sourceTag: config.sourceTag, processingType: processingType, data: row };
+        process.stdout.write("#");
+        request.json = true;
+
+        logger.log('debug', "sending data:" + JSON.stringify(outData));
+        request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(outData), json: true }).then(_ => {
+            // count up the maxPKey only when the request was successful...
+            if (row.PKey > maxPKey) {
+                maxPKey = row.PKey;
+                fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
+                logger.log("debug", "MaxPKey Written: " + maxPKey);
+            }
+        });
+    });
+
+    sqlRequest.on('done', result => {
+        if (lastOutMaxPKey !== maxPKey) {
+            process.stdout.write("[" + maxPKey + "]\n");
+            lastOutMaxPKey = maxPKey;
+        } else {
+            process.stdout.write(".");
+        }
+    });
+})
--- a/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat
+++ b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat
@@ -0,0 +1 @@
+0010000000a87kiz
--- a/index.js
+++ b/index.js
@@ -1,35 +0,0 @@
-const micro = require('micro')
-const {json,send} = require('micro')
-
-const reTest= (regex,data)=>{
-    let match = regex.match(new RegExp('^/(.*?)/([giumy]*)$'));
-    regex = new RegExp(match[1], match[2]);
-    return regex.test(data);
-}
-
-// SourceTag
-// ProcessingType
-
-const server = micro(async (req,res) => {
-    //console.log(req);
-    const js = await json(req);
-    //console.log("Received Data",js); 
-
-    try {
-        inputLine = JSON.parse(js);
-    } catch (ex) {
-        console.error("Input not in JSON format:" + line);
-    }
-
-    if (inputLine.processingType==="casrpslog") {
-        if (inputLine.data.Severity > 1 ) {
-            console.log("Issue ",inputLine.data.LogText);
-        }
-    }
-    //console.log("line: " + reTest("/error/i",inputLine));
-
-
-    send(res,200,"OK");
-});
-
-server.listen(2222);
--- a/logAggregator.js
+++ b/logAggregator.js
@@ -0,0 +1,49 @@
+const micro = require('micro');
+const { json, send } = require('micro');
+const { RuleEngine } = require('./RuleEngine');
+const fs = require('fs');
+const jsonminify = require('jsonminify');
+const winston = require("winston");
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => {
+            return `${info.timestamp} ${info.level}: ${info.message}`;
+        })
+    ),
+    transports: [
+        new winston.transports.Console(),
+        new winston.transports.File({ filename: './log/LogAggregatorError.log', level: 'error' }),
+        new winston.transports.File({ filename: './log/Combined.log' })
+    ]
+});
+
+
+const ruleEngine = new RuleEngine("casrpslog");
+logger.log("info","Service Starting.");
+const server = micro(async (req, res) => {
+    const js = await json(req);
+
+    try {
+        inputLine = JSON.parse(js);
+    } catch (ex) {
+        logger.log("error","Input not in JSON format:" + line);
+    }
+
+    let isWhiteListed = ruleEngine.processData(inputLine);
+    let miniInput=jsonminify(JSON.stringify(inputLine)) ;
+    let now = Date();
+    if (isWhiteListed) {
+        logger.log("debug","Not Whitelisted: " + miniInput);
+        fs.appendFileSync("./log/skipped.log",miniInput + "\n");
+    } else {
+        logger.log("info","Not Whitelisted: " + miniInput);
+        fs.appendFileSync("./log/issues.log",miniInput + "\n");
+    } 
+
+    send(res, 200, "OK");
+});
+
+server.listen(2222);
--- a/package-lock.json
+++ b/package-lock.json
@@ -41,6 +41,14 @@
      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
    },
+    "async": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
+      "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
+      "requires": {
+        "lodash": "^4.17.10"
+      }
+    },
    "asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -112,6 +120,15 @@
        "supports-color": "^5.3.0"
      }
    },
+    "color": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
+      "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
+      "requires": {
+        "color-convert": "^1.9.1",
+        "color-string": "^1.5.2"
+      }
+    },
    "color-convert": {
      "version": "1.9.3",
      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -125,6 +142,34 @@
      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
    },
+    "color-string": {
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz",
+      "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==",
+      "requires": {
+        "color-name": "^1.0.0",
+        "simple-swizzle": "^0.2.2"
+      }
+    },
+    "colornames": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz",
+      "integrity": "sha1-+IiQMGhcfE/54qVZ9Qd+t2qBb5Y="
+    },
+    "colors": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.3.3.tgz",
+      "integrity": "sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg=="
+    },
+    "colorspace": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.1.tgz",
+      "integrity": "sha512-pI3btWyiuz7Ken0BWh9Elzsmv2bM9AhA7psXib4anUXy/orfZ/E0MbQwhSOG/9L8hLlalqrU0UhOuqxW1YjmVw==",
+      "requires": {
+        "color": "3.0.x",
+        "text-hex": "1.0.x"
+      }
+    },
    "combined-stream": {
      "version": "1.0.7",
      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
@@ -174,6 +219,16 @@
      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
      "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
    },
+    "diagnostics": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz",
+      "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==",
+      "requires": {
+        "colorspace": "1.1.x",
+        "enabled": "1.0.x",
+        "kuler": "1.0.x"
+      }
+    },
    "ecc-jsbn": {
      "version": "0.1.2",
      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -183,6 +238,19 @@
        "safer-buffer": "^2.1.0"
      }
    },
+    "enabled": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz",
+      "integrity": "sha1-ll9lE9LC0cX0ZStkouM5ZGf8L5M=",
+      "requires": {
+        "env-variable": "0.0.x"
+      }
+    },
+    "env-variable": {
+      "version": "0.0.5",
+      "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz",
+      "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA=="
+    },
    "escape-string-regexp": {
      "version": "1.0.5",
      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
@@ -208,6 +276,16 @@
      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
    },
+    "fast-safe-stringify": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz",
+      "integrity": "sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg=="
+    },
+    "fecha": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz",
+      "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg=="
+    },
    "forever-agent": {
      "version": "0.6.1",
      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -296,6 +374,11 @@
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
    },
+    "is-arrayish": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
+      "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
+    },
    "is-stream": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
@@ -336,6 +419,11 @@
      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
    },
+    "jsonminify": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/jsonminify/-/jsonminify-0.4.1.tgz",
+      "integrity": "sha1-gF2vuzk5UYjO6atYLIHvlZ1+cQw="
+    },
    "jsprim": {
      "version": "1.4.1",
      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
@@ -347,11 +435,31 @@
        "verror": "1.10.0"
      }
    },
+    "kuler": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz",
+      "integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==",
+      "requires": {
+        "colornames": "^1.1.1"
+      }
+    },
    "lodash": {
      "version": "4.17.11",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
    },
+    "logform": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz",
+      "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==",
+      "requires": {
+        "colors": "^1.2.1",
+        "fast-safe-stringify": "^2.0.4",
+        "fecha": "^2.3.3",
+        "ms": "^2.1.1",
+        "triple-beam": "^1.2.0"
+      }
+    },
    "micro": {
      "version": "9.3.3",
      "resolved": "https://registry.npmjs.org/micro/-/micro-9.3.3.tgz",
@@ -407,6 +515,11 @@
      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
    },
+    "one-time": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
+      "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4="
+    },
    "performance-now": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -535,6 +648,14 @@
      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
      "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
    },
+    "simple-swizzle": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
+      "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=",
+      "requires": {
+        "is-arrayish": "^0.3.1"
+      }
+    },
    "sprintf-js": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz",
@@ -556,6 +677,11 @@
        "tweetnacl": "~0.14.0"
      }
    },
+    "stack-trace": {
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
+      "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
+    },
    "statuses": {
      "version": "1.5.0",
      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
@@ -598,6 +724,11 @@
        "sprintf-js": "^1.1.1"
      }
    },
+    "text-hex": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
+      "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
+    },
    "tough-cookie": {
      "version": "2.4.3",
      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
@@ -614,6 +745,11 @@
        }
      }
    },
+    "triple-beam": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
+      "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw=="
+    },
    "tunnel-agent": {
      "version": "0.6.0",
      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
@@ -659,6 +795,31 @@
        "core-util-is": "1.0.2",
        "extsprintf": "^1.2.0"
      }
+    },
+    "winston": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.1.0.tgz",
+      "integrity": "sha512-FsQfEE+8YIEeuZEYhHDk5cILo1HOcWkGwvoidLrDgPog0r4bser1lEIOco2dN9zpDJ1M88hfDgZvxe5z4xNcwg==",
+      "requires": {
+        "async": "^2.6.0",
+        "diagnostics": "^1.1.1",
+        "is-stream": "^1.1.0",
+        "logform": "^1.9.1",
+        "one-time": "0.0.4",
+        "readable-stream": "^2.3.6",
+        "stack-trace": "0.0.x",
+        "triple-beam": "^1.3.0",
+        "winston-transport": "^4.2.0"
+      }
+    },
+    "winston-transport": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz",
+      "integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==",
+      "requires": {
+        "readable-stream": "^2.3.6",
+        "triple-beam": "^1.2.0"
+      }
    }
  }
 }
--- a/package.json
+++ b/package.json
@@ -2,18 +2,21 @@
  "name": "logaggregator",
  "version": "1.0.0",
  "description": "",
-  "main": "index.js",
+  "main": "logAggregator.js",
  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node logAggregator.js"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
+    "jsonminify": "^0.4.1",
    "micro": "^9.3.3",
    "msnodesqlv8": "^0.6.11",
    "mssql": "^4.3.0",
    "readline": "^1.3.0",
    "request": "^2.88.0",
-    "request-promise": "^4.2.2"
+    "request-promise": "^4.2.2",
+    "winston": "^3.1.0"
  }
 }
--- a/rules/casroslogfull.rule.json
+++ b/rules/casroslogfull.rule.json
@@ -0,0 +1,26 @@
+{ "casrpslog":{ 
+    "rules":[
+        {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"},
+        {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
+        {"Logic":"&", "rules":[
+            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
+            {"Logic":"|", "rules":[
+              {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
+              {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
+            ]}
+          ] 
+        },
+        {"Logic":"&", "rules":[
+          {"Logic":"|", "rules":[
+            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
+            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"}
+          ]},
+          {"Logic":"|", "rules":[
+            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."},
+            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."},
+            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"}
+          ]}
+        ]}
+      ]
+    }
+  }
--- a/rules/casrpslog.rule.json
+++ b/rules/casrpslog.rule.json
@@ -0,0 +1,27 @@
+{ "casrpslog":{ 
+  "rules":[
+      {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, 
+      {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
+      {"Logic":"&", "rules":[
+          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
+          {"Logic":"|", "rules":[
+            {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
+            {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
+          ]}
+        ] 
+      },
+      {"Logic":"&", "rules":[
+        {"Logic":"|", "rules":[
+          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
+          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"},
+          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationInboundWorker"}
+        ]},
+        {"Logic":"|", "rules":[
+          {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."},
+          {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."},
+          {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"}
+        ]}
+      ]}
+    ]
+  }
+}