logAggregator/casRpsLogExtractor/casRpsLogExtractor.js

const sql = require("mssql/msnodesqlv8");
//const sql = require("mssql");
const request = require("request-promise");
const fs = require("fs");
const winston = require("winston");

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.printf(info => {
            return `${info.timestamp} ${info.level}: ${info.message}`;
        })
    ),
    transports: [
        new winston.transports.Console(),
        new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }),
        new winston.transports.File({ filename: './log/Combined.log' })
    ]
});

console.log(`casRpsLogExtractor Copyright (C) 2019  j.tretter@gmail.com
    
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.`);

logger.log("info", `casRpsLogExtractor v0.1 Starting`);

const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8"));

logger.log("debug", "Config" + JSON.stringify(config))
const processingType = "casrpslog";
const maxResultsPerMessage=100;
let maxPKey = "";
let maxPKeyUnsent = "";
let allRows = [];

let theInterval;


try {
    lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
    logger.log("debug", "MaxPKey Loaded: " + maxPKey);
} catch (ex) {
    if (ex.errno === -4058) {
        logger.log("info", "MaxPKey-File not existing. Will be created.");
    } else {
        logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex);
    }
}

sql.connect(config.sqlConfig, err => {
    if (err) {
        logger.log("error", "Connecting to DB: " + err);
    }
    const sqlRequest = new sql.Request();
    sqlRequest.stream = true;

	function runNextPoll(){
		sqlRequest.query("select top " + maxResultsPerMessage + " * from rpslog where pkey > '" + maxPKey + "' order by pkey");
	}
	
    runNextPoll();

    sqlRequest.on('row', row => {
        outData = { sourceTag: config.sourceTag, processingType: processingType, data: row };
        process.stdout.write("#");
        request.json = true;

        logger.log('debug', "adding row to array:" + JSON.stringify(outData));
        allRows.push(outData);

        if (row.PKey > maxPKeyUnsent) {
            maxPKeyUnsent = row.PKey;
            logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent);
        }
    });

    sqlRequest.on('done', result => {
		clearInterval(theInterval);
        if (allRows.length>0){
            request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
                // count up the maxPKey only when the request was successful...
                maxPKey=maxPKeyUnsent;
                fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
            }).catch(err=>{
                logger.log("error","Sending Request to Service: " + err)
            }).finally(_=>{
				runNextPoll();
			});
            process.stdout.write(">" + allRows.length + ">");
		} else {
			theInterval = setInterval(runNextPoll, config.pollIntervalIdle);
			process.stdout.write(".");
        }
        allRows=[];

    });
})