Merge pull request #25 from w3f/ibm

Ibm

README.md

@@ -2,4 +2,26 @@
 
 # virustotal-prometheus
 
-Monitor VirusTotal reports and expose Prometheus metrics
+App to monitor (mainly) VirusTotal reports and to expose Prometheus metrics.  
+
+The application is K8s ready, and it provides also a ServiceMonitor and a PrometheusRule configurations that can be used by your Prometheus/Alertmanager.
+
+## Intelligence sources
+
+- VirusTotal
+- IBM Xforce (optional)
+
+## Domain List
+
+It can be configured via a [config file](/config/main.sample.yaml). 
+
+### Api sources for the domain list
+- Cloudflare (optional):  
+the domain list can be dynamically enriched via Clodflare. Please set up this connection with a read-only apiKey.
+
+## How to Run
+
+```
+yarn
+yarn start -c path_to_config_file
+```

charts/virustotal-prometheus/Chart.yaml

@@ -1,5 +1,5 @@
 description: Virustotal Prometheus
 name: virustotal-prometheus
-version: v0.2.0
-appVersion: v0.2.0
+version: v0.3.0
+appVersion: v0.3.0
 apiVersion: v2

charts/virustotal-prometheus/templates/alertrules.yaml

@@ -23,5 +23,15 @@ spec:
       for: 2m
       labels:
         severity: warning
-        origin: {{ .Values.prometheusRules.origin }}     
+        origin: {{ .Values.prometheusRules.origin }} 
+    {{ if eq .Values.config.ibmXforce.enabled true }}    
+    - alert: IbmXforceBadScore
+      annotations:
+        message: 'Domain {{`{{ $labels.domain }}`}} might have been flagged as problematic, please visit https://www.exchange.xforce.ibmcloud.com/url/{{`{{ $labels.domain }}`}} to doublecheck.'
+      expr: max without(instance,pod) (last_over_time(ibm_xforce_score[10m])) > 1
+      for: 2m
+      labels:
+        severity: warning
+        origin: {{ .Values.prometheusRules.origin }}
+    {{ end }}       
 {{ end }}

charts/virustotal-prometheus/values.yaml

@@ -16,6 +16,10 @@ config:
       - example.com
   virusTotal:
     apiKey: secret
+  ibmXforce:
+    enabled: false
+    apiKey: xxx
+    apiPassword: xxx    
 
 serviceMonitor:
   enabled: true

config/main.sample.yaml

@@ -8,4 +8,8 @@ targetDomains:
   manualList:  
   - example.com
 virusTotal:
-  apiKey: secret
+  apiKey: secret
+ibmXforce:
+  enabled: false
+  apiKey: xxx
+  apiPassword: xxx    

package.json

@@ -1,6 +1,6 @@
 {
   "name": "virustotal-prometheus",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Monitor VirusTotal reports and expose Prometheus metrics",
   "repository": "[email protected]:w3f/report-scanner.git",
   "author": "W3F Infrastructure Team <[email protected]>",
@@ -25,6 +25,7 @@
     "cloudflare": "^2.9.1",
     "commander": "^4.0.0",
     "express": "^4.18.1",
+    "got": "^11",
     "node-virustotal": "^3.35.0",
     "prom-client": "^14.0.1"
   },

src/actions/start.ts

@@ -6,59 +6,73 @@ import { register } from 'prom-client';
 import { InputConfig } from '../types';
 import { evalIntervalMinutes } from '../constants';
 import vt from 'node-virustotal'
-import Cloudflare = require('cloudflare')
+import cloudflare from 'cloudflare'
+import got from 'got'
 
-const logger = LoggerSingleton.getInstance()
+interface LookupConfig {
+    vtApi: any;
+    ibmApiKey?: string;
+    ibmPassword?: string;
+    ibmEnabled: boolean;
+    promClient: Prometheus;
+}
 
+let logger = LoggerSingleton.getInstance()
 
-function lookup(vtApi: any, promClient: Prometheus, domain: string){
-    logger.debug(`triggering a lookup for the domain ${domain}`)
+function vtLookup(vtApi: any, promClient: Prometheus, domain: string): void {
+    logger.debug(`triggering a Virustotal lookup for the domain ${domain}`)
     vtApi.domainLookup(domain,function(err, res){
         if (err) {
-        logger.error(`Error on processing ${domain}`);
+        logger.error(`Error on VT processing ${domain}`);
         logger.error(err);
         process.exit(-1)
         }
 
         const parsed = JSON.parse(res)
         const reports =  parsed.data.attributes.last_analysis_stats.malicious + parsed.data.attributes.last_analysis_stats.suspicious
         logger.info(`${domain} reports: ${reports}`);
-        promClient.setVTReports(domain,reports)
-        return;
+        promClient.setVTReport(domain,reports)
+        logger.debug(`VT lookup for the domain ${domain} DONE`)
     })
-    logger.debug(`lookup for the domain ${domain} DONE`)
 }
 
-export async function startAction(cmd): Promise<void> {
+async function ibmLookup(apiKey: string, apiPassword: string ,promClient: Prometheus, domain: string): Promise<void> {
+    logger.debug(`triggering an IBM lookup for the domain ${domain}`)
+    const url = `https://api.xforce.ibmcloud.com/api/url/${domain}`;
 
-    const cfg = new Config<InputConfig>().parse(cmd.config);
-    LoggerSingleton.setInstance(cfg.logLevel)
-
-    const domainsSet = new Set<string>()
-    if(cfg.targetDomains.cloudflare.enabled){
-        const cf = new Cloudflare({
-            token: cfg.targetDomains.cloudflare.apiKey
-        });
+    const options = {
+        headers: {
+            'accept': 'application/json',
+            'Authorization': `Basic ${Buffer.from(`${apiKey}:${apiPassword}`).toString("base64")}`
+        },
+    };
 
-        try {
-            const zones = await cf.zones.browse()
-            zones.result.forEach(element => {
-                domainsSet.add(element.name)
-            });
-        } catch (error) {
-            logger.error(`Error on processing the clouflare api`);
+    try {
+        const response: any = await got(url,options).json();
+        logger.debug(JSON.stringify(response))
+        const score: number = response.result.score? response.result.score : 1 //unkown is treated as OK
+        logger.info(`${domain} ibm score, 1 is OK: ${score}`);
+        promClient.setIbmScore(domain,score)
+        logger.debug(`IBM lookup for the domain ${domain} DONE`)
+    } catch (error) {
+        const errorMessage: string = error.toString()
+        if(errorMessage.includes("404")){
+            logger.warn(`IBM api is not capable of processing ${domain}`)
+            logger.debug(errorMessage);
+        } else{
+            logger.error(`Error on IBM processing ${domain}`);
             logger.error(error);
             process.exit(-1)
         }
     }
+}
 
-    const manualList = cfg.targetDomains.manualList ? cfg.targetDomains.manualList : []
-    manualList.forEach(domain=>domainsSet.add(domain))
-    const targetDomains = Array.from( domainsSet.values() )
+async function lookup(config: LookupConfig, domain: string): Promise<void> {
+    vtLookup(config.vtApi,config.promClient,domain)
+    if(config.ibmEnabled) await ibmLookup(config.ibmApiKey,config.ibmPassword,config.promClient,domain)
+}
 
-    logger.debug(targetDomains.toString())
-
-    const server = express();
+function configureServerEndpoints(server: express.Express): void {
     server.get('/healthcheck',
         async (req: express.Request, res: express.Response): Promise<void> => {
             res.status(200).send('OK!')
@@ -67,17 +81,59 @@ export async function startAction(cmd): Promise<void> {
             res.set('Content-Type', register.contentType)
             res.end(await register.metrics())
         })    
-    server.listen(cfg.port);
+}
+
+async function cfAddDomains(apiKey: string, domains: Set<string>): Promise<void> {
+    const cf = cloudflare({
+        token: apiKey
+    });
+
+    try {
+        const zones = await cf.zones.browse()
+        zones.result.forEach(element => {
+            domains.add(element.name)
+        });
+    } catch (error) {
+        logger.error(`Error on processing the clouflare api`);
+        logger.error(error);
+        process.exit(-1)
+    }
+}
+
+export async function startAction(cmd): Promise<void> {
+
+    const cfg = new Config<InputConfig>().parse(cmd.config);
+    logger = LoggerSingleton.getNewInstance(cfg.logLevel)
+
+    const domainsSet = new Set<string>()
+    if(cfg.targetDomains.cloudflare.enabled){
+        await cfAddDomains(cfg.targetDomains.cloudflare.apiKey,domainsSet)
+    }
+    const manualList = cfg.targetDomains.manualList ? cfg.targetDomains.manualList : []
+    manualList.forEach(domain=>domainsSet.add(domain))
+    const targetDomains = Array.from( domainsSet.values() )
+    logger.info(`Target List: ${targetDomains.toString()}`)
 
-    const api = vt.makeAPI();
-    api.setKey(cfg.virusTotal.apiKey)
+    const server = express();
+    configureServerEndpoints(server)
+    server.listen(cfg.port);
+
     const promClient = new Prometheus();
     const evalInterval = cfg.evalIntervalMinutes? cfg.evalIntervalMinutes*1000*60 : evalIntervalMinutes
-    targetDomains.forEach(domain=>lookup(api,promClient,domain)) 
+
+    const vtApi = vt.makeAPI();
+    vtApi.setKey(cfg.virusTotal.apiKey)
+
+    const lookupConfig = {
+        vtApi: vtApi,
+        ibmApiKey: cfg.ibmXforce.apiKey,
+        ibmPassword: cfg.ibmXforce.apiPassword,
+        ibmEnabled: cfg.ibmXforce.enabled,
+        promClient: promClient
+    }
+    targetDomains.forEach(domain => lookup(lookupConfig,domain)) 
     setInterval(
-        () => targetDomains.forEach(domain => {
-            lookup(api,promClient,domain)
-        }),
+        () => targetDomains.forEach(domain => lookup(lookupConfig,domain)),
         evalInterval
     )
 }

src/logger.ts

@@ -5,15 +5,16 @@ export class LoggerSingleton {
     private constructor() {
         //do nothing
     }
-    public static setInstance(level: string): void {
-        LoggerSingleton.instance = createLogger(level)
-    }
     public static getInstance(level?: string): Logger {
         if (!LoggerSingleton.instance) {
             LoggerSingleton.instance = createLogger(level)
         }        
         return LoggerSingleton.instance
     }
+    public static getNewInstance(level?: string): Logger {
+        LoggerSingleton.instance = createLogger(level)      
+        return LoggerSingleton.instance
+    }
 }
 
 export type Logger = LoggerW3f

src/prometheus.ts

@@ -5,6 +5,7 @@ import { PromClient } from './types';
 export class Prometheus implements PromClient {
 
     private virustotalReports: promClient.Gauge<"domain">;
+    private ibmScore: promClient.Gauge<"domain">;
 
     private readonly logger: Logger = LoggerSingleton.getInstance()
 
@@ -21,15 +22,23 @@ export class Prometheus implements PromClient {
         promClient.collectDefaultMetrics();
     }
 
-    setVTReports(domain: string, reports: number): void{
+    setVTReport(domain: string, reports: number): void{
       this.virustotalReports.set({domain}, reports);        
     }
+    setIbmScore(domain: string, score: number): void{
+        this.ibmScore.set({domain}, score);        
+      }
 
     _initMetrics(): void {
         this.virustotalReports = new promClient.Gauge({
             name: 'virustotal_reports',
             help: 'Virustotal bad status report for a specific domain',
             labelNames: ['domain']
         });
+        this.ibmScore = new promClient.Gauge({
+            name: 'ibm_xforce_score',
+            help: 'Ibm xforce score for a specific domain, 1 is good',
+            labelNames: ['domain']
+        });
     }
 }

src/types.ts

@@ -13,8 +13,14 @@ export interface InputConfig {
     virusTotal: {
         apiKey: string;
     };
+    ibmXforce: {
+        enabled: boolean;
+        apiKey: string;
+        apiPassword: string;
+    };
 }
 
 export interface PromClient {
-    setVTReports(domain: string, reports: number): void;
+    setVTReport(domain: string, reports: number): void;
+    setIbmScore(domain: string, score: number): void;
 }