Fortify FVDL Files – First working Parser and Viewer for *.fvdl files

Following from the previous Fortify FVDL posts (here,  here, here and here), here is a first working tool that is able to load up *.fvdl files, parse its relevant data into a new set of classes, and then visuzalize its contents into a number of tabs.

This is what the main GUI look like (after loading/dropping the 35Mb dspace.fvdl file):

On the left there is a PropertyGrid control that shows a view of the new O2 Fortify_Scan class, and on the right there is a TabControl with a number of Tab Pages (each containing a raw view of the classes created).

Here is what each of the  tab pages looks like:

ScannedFiles

Contexts:

Descriptions

CalledWithNoDefs

Sinks

Sources

Snippets

Vulnerabilities

 The next post will show what the updated API_Fortify looks like, and the code that was used to create this GUI

Fortify FVDL files – Creating .NET classes that map to Fvdl xml structure

I’m still getting my head around how the *.fvdl files are structured, but after looking at the data that they seem to contains (using the fvdl.details() viewer), I’ve created the following classes which (I think) represent the type of data that is contained in the fvdl files (I tried to consolidate the data structures a bit, but I’m sure there is still quite a bit of refectoring and optimization that can be done):

    public class Fortify_Scan
    {
        public FVDL _fvdl;
       
        public string BuildID { get; set; }
        public string LOC { get; set; }
        public string SourceBasePath { get; set; }
        public string CreatedDate { get; set; }
        public string CreatedTime { get; set; }       
        public List<Fortify_Vulnerability> Vulnerabilities { get; set; }
        public List<Fortify_Context> Contexts { get; set; }
        public List<Fortify_Description> Descriptions { get; set; }
        public List<Fortify_Sink> Sinks { get; set; }
        public List<Fortify_Source> Sources { get; set; }
       
        public Fortify_Scan()
        {
            Vulnerabilities = new List<Fortify_Vulnerability>();
            Contexts = new List<Fortify_Context>();
            Descriptions = new List<Fortify_Description>();
            Sinks = new List<Fortify_Sink>();
            Sources = new List<Fortify_Source>();
        }
    }       
           
    public class Fortify_Context
    {
        public string Id { get; set; }
        public Fortify_Function Function { get; set; }
    }
   
    public class Fortify_Function
    {
        public string FunctionName { get; set; }
        public Fortify_CodeLocation CodeLocation { get; set; }       
    }
   
    public class Fortify_CodeLocation
    {
        public string Path { get; set; }
        public string Line { get; set; }
        public string LineEnd { get; set; }
        public string ColStart { get; set; }
        public string ColEnd { get; set; }
    }
   
    public class Fortify_Description
    {
        public string Abstract { get; set; }
        public string ClassID { get; set; }
        public string ContentType { get; set; }
        public string Explanation { get; set; }
        public string Recommendations { get; set; }
        public string Tips { get; set; }       
    }
   
    public class Fortify_Sink
    {
        public string ruleID { get; set; }
        public Fortify_Function Function_Call{ get; set;}               
    }
   
    public class Fortify_Source
    {
        public string ruleID { get; set; }
        public Fortify_Function Function_Call{ get; set;}               
        public Fortify_Function Function_Entry{ get; set;}               
        public List<string> TaintFlags { get; set;}
        public Fortify_Source()
        {
            TaintFlags = new List<string>();
        }
    }       
   
    public class Fortify_Snippet
    {
        public string Id { get; set; }
        public Fortify_CodeLocation CodeLocation { get; set; }       
        public string Text { get; set;}       
    }
    public class Fortify_TraceEntry
    {
        public Fortify_TraceEntryNode Node         { get; set; }
        public string NodeRef                     { get; set; }
    }
    public class Fortify_TraceEntryNode
    {
        public bool IsDefault                             { get; set; }
        public string ActionType                         { get; set; }
        public string ActionValue                         { get; set; }
        public Fortify_CodeLocation SourceLocation         { get; set; }
        public Fortify_CodeLocation SecundaryLocation     { get; set; }
        public Fortify_Snippet Snippet                     { get; set; }
        public string ContextId                            { get; set; }
        public string ReasonRuleId                        { get; set; }
        public string Label                                { get; set; }
        public List<Fortify_TraceEntryFact> Facts        { get; set; }
       
    }
   
    public class Fortify_TraceEntryFact
    {
        public bool Primary {get;set;}
        public string Type {get;set;}
        public string Value {get;set;}
    }
    public class Fortify_Vulnerability
    {
        public string Kingdom { get; set; }                // from ClassInfo
        public string Analyzer { get; set; }
        public string ClassId { get; set; }
        public decimal DefaultSeverity { get; set; }
        public string  Type { get; set; }
        public string  SubType { get; set; }
       
        public decimal Confidence { get; set; }            // from InstanceInfo
        public string InstanceId { get; set; }
        public decimal InstanceSeverity { get; set; }
       
       
        public Fortify_Function Function { get; set; }       
        public Items ReplacementDefinitions { get; set; }
        public List<Fortify_TraceEntry> Traces { get; set; }
    }

(also, the data structures that I’m seeing are directly mapped to the object that was created from the current XSD/C# file, so if you know the inner structure of the *.fvdl files and see missing bits of data that are very useful to have or visualize, please let me know)

To reflect the new Classes (and the fact that the main object is now the Fortify_Scan class), I’ve modified the API_Fortify class (note that there is still quite a bit to go, since the mapping functions are just importing some of the vulnerability data available)

    public class API_Fortify
    {               
        public Fortify_Scan convertToFortifyScan(string fvdlFile)
        {
            var scan = new Fortify_Scan();
            scan._fvdl = loadFvdl_Raw(fvdlFile);
            scan.mapFvdlData();
            return scan;
        }
       
        public FVDL loadFvdl_Raw(string fvdlFile)
        {
            try
            {
                var chachedFvdl = (FVDL)O2LiveObjects.get(fvdlFile);
                if (chachedFvdl.notNull())
                    return chachedFvdl;
            }
            catch { }
            
            var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();       
             var _fvdl = FVDL.Load(fvdlFile);   
             O2LiveObjects.set(fvdlFile,_fvdl);
             o2Timer.stop();
             return _fvdl; 
        }
    }
   
    public static class Fortify_Scan_ExtensionMethods_MappingFvdl
    {
        public static Fortify_Scan mapFvdlData(this Fortify_Scan fortifyScan)
        {
            fortifyScan.mapVulnerabilities();
            return fortifyScan;
        }
       
        public static Fortify_Scan mapVulnerabilities(this Fortify_Scan fortifyScan)
        {                
                foreach(var vulnerability in fortifyScan._fvdl.Vulnerabilities.Vulnerability)
                {
                      var fortifyVulnerability = new Fortify_Vulnerability
                              {
                                Kingdom = vulnerability.ClassInfo.Kingdom,
                                Analyzer = vulnerability.ClassInfo.AnalyzerName,
                                ClassId = vulnerability.ClassInfo.ClassID,
                                DefaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                InstanceId = vulnerability.InstanceInfo.InstanceID,
                                InstanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                Confidence = vulnerability.InstanceInfo.Confidence,                                          
/*                                Function = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.Function.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.Function.name
                                            : "" ,
                                File = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path
                                            : "" ,
                                Line = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line
                                            : 0
*/                                           
                            };
                    fortifyScan.Vulnerabilities.add(fortifyVulnerability);   
                };                               
                return fortifyScan;
        }
    }       

Here is the updated viewer (that now consumes a Fortify_Scan object)

var topPanel = panel.clear().add_Panel();
//var topPanel = "Util - Simple FVDL viewer".popupWindow(1000,400);</pre>
&nbsp;

var tableList = topPanel.clear().add_TableList().title("Drop an *.fvdl file here to load it");
var propertyGrid = topPanel.insert_Left().add_PropertyGrid();
var apiFortify = new API_Fortify();

Action<List<Fortify_Vulnerability>> showFvdl =
    (vulnerabilities) =>
        {
                 tableList.title("Showing {0} Vulnerabilties".format(vulnerabilities.size()))
                         .show(vulnerabilities);                                                        
        };
 
Action<string> loadAndShowFile =
    (file)=>{
                tableList.title("... loading file: {0}".format(file.fileName()));
                O2Thread.mtaThread(()=>{                                                                                       
                                            var fortifyScan = apiFortify.convertToFortifyScan(file);
                                            showFvdl(fortifyScan.Vulnerabilities);       
                                            propertyGrid.show(fortifyScan);
                                        });
            };

tableList.onDrop(loadAndShowFile);
tableList.getListViewControl().onDrop(loadAndShowFile);
   
var testFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
loadAndShowFile(testFile);
/*(var _fortifyScan = apiFortify.convertToFortifyScan(xmlFile);
propertyGrid.show(_fortifyScan);   
showFvdl(_fortifyScan.Vulnerabilities);*/

return "done"; 

//O2File:C:\O2\O2Scripts_Database\_Scripts\3rdParty_Tools\Fortify\API_Fortify_1_6.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

Note that I added a PropertyGrid to the left of the GUI which will show the contents of the Fortity_Scan object

Fortify FVDL files – Creating an API and consumining it

Following from the (Fortify FVDL related) Creating and consuming the schema and CSharp file and Simple TableList Viewer Tool posts, this one shows the next evolutionary step, which is the creation of an API that can be easily consumed by *.h2 scripts.

The script show here will have the same funcionality has the one shown in Simple TableList Viewer Tool, but the its structure will be completely different.

 The data will be stored in a dedicated class (previously we used an anonymous class)

public class Fortify_Vulnerability
    {
        public string Kingdom { get; set; }
        public string Analyzer { get; set; }
        public string ClassId { get; set; }
        public decimal DefaultSeverity { get; set; }
        public string InstanceId { get; set; }
        public decimal InstanceSeverity { get; set; }
        public decimal Confidence { get; set; }
        public string Function { get; set; }
        public string File { get; set; }
        public int Line { get; set; }
    }

And the main functions of loading and parsing are now exposed in an API file called API_Fortify_1_6.cs (which provides the class API_Fortify):

    public class API_Fortify
    {
        public FVDL loadFvdl(string fvdlFile)
        {
            try
            {
                var chachedFvdl = (FVDL)O2LiveObjects.get(fvdlFile);
                if (chachedFvdl.notNull())
                    return chachedFvdl;
            }
            catch { }
            
            var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();       
             var _fvdl = FVDL.Load(fvdlFile);   
             O2LiveObjects.set(fvdlFile,_fvdl);
             o2Timer.stop();
             return _fvdl; 
        }
       
       
        public List<Fortify_Vulnerability> getVulnerabilities(FVDL fvdl)
        {
                
                 var fortifyVulnerabities = new List<Fortify_Vulnerability>();               
                foreach(var vulnerability in fvdl.Vulnerabilities.Vulnerability)
                {
                      var fortifyVulnerability = new Fortify_Vulnerability
                              {
                                Kingdom = vulnerability.ClassInfo.Kingdom,
                                Analyzer = vulnerability.ClassInfo.AnalyzerName,
                                ClassId = vulnerability.ClassInfo.ClassID,
                                DefaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                InstanceId = vulnerability.InstanceInfo.InstanceID,
                                InstanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                Confidence = vulnerability.InstanceInfo.Confidence,                                          
                                Function = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.Function.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.Function.name
                                            : "" ,
                                File = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path
                                            : "" ,
                                Line = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line
                                            : 0
                            };
                    fortifyVulnerabities.add(fortifyVulnerability);   
                };
                return fortifyVulnerabities;
        }
    }       

The GUI script is now much smaller and is mailly focused on creating the GUI and consuming the API_Fortify class:

//var topPanel = panel.clear().add_Panel();
var topPanel = "Util - Simple FVDL viewer".popupWindow(1000,400);
var tableList = topPanel.clear().add_TableList().title("Drop an *.fvdl file here to load it");

var apiFortify = new API_Fortify();

Action<List<Fortify_Vulnerability>> showFvdl =
    (vulnerabilities) =>
        {
                 tableList.title("Showing {0} Vulnerabilties".format(vulnerabilities.size()))
                         .show(vulnerabilities);                                                        
        };
 
Action<string> loadAndShowFile =
    (file)=>{
                tableList.title("... loading file: {0}".format(file.fileName()));
                O2Thread.mtaThread(()=>{
                                            var fvdl = apiFortify.loadFvdl(file);
                                            showFvdl(apiFortify.getVulnerabilities(fvdl));
                                        });
            };

tableList.onDrop(loadAndShowFile);
tableList.getListViewControl().onDrop(loadAndShowFile);
   
//var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
//var _fvdl = apiFortify.loadFvdl(xmlFile);
//showFvdl(apiFortify.getVulnerabilities(_fvdl));

return "done"; 

//O2File:C:\O2\O2Scripts_Database\_Scripts\3rdParty_Tools\Fortify\API_Fortify_1_6.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

The end result is the same as with the script shown in the  Simple TableList Viewer Tool post:

Fortify FVDL files – Simple TableList Viewer Tool

Following on from the Fortify FVDL files – Creating and consuming the schema and CSharp file  post , let’s now build a generic simple tool to view fvdl files (which as long as they are compliant with the XSD we created, they should load).

Note: These scripts are going to use the demo files referenced in the previous post, and that you can download from http://s3.amazonaws.com/Demo_Files/Fortify-Sate-2008.zip . This zip should had been unziped to the ‘C:\O2\_tempDir\_Fortify-Sate-2008\’ folder (as per the previous scripts) and the C# that I’m going to use is the one that you will find at ‘C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\Fortify.fvdl.1.6.cs’ (this is the same one as created by the previous example, except that is located on a different folder and has a different name)

The first step is to load up a file and view it in a ListView (this is the last example of the previous script)

var topPanel = panel.clear().add_Panel();
var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
var vulnerabilities = fvdl.Vulnerabilities.Vulnerability; 

var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,
                                function = vulnerability.AnalysisInfo.Unified.Context.Function.name,
                                file = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path,
                                line = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line,
                                col = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.colStart
                               
                            }).toList();

topPanel.add_TableList("Showing {0} Vulnerabilties".format(results.size()))
        .show(results);                           
return "done";

//using xmlns.www.fortifysoftware.com.schema.fvdl
//O2File:C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\Fortify.fvdl.1.6.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

 

The first thing to do is to move the loading of the fvdl into a separate Lamdba method:

Func<string, FVDL> loadFvdl =
    (fvdlFile)=>{
                    var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();       
                     var _fvdl = FVDL.Load(fvdlFile);   
                     o2Timer.stop();
                     return _fvdl;
                };
 
var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = loadFvdl(xmlFile);

Then also  move the code that shows the results into its own Lambda function

Action<FVDL> showFvdl =
     (_fvdl)=>{
                var vulnerabilities = _fvdl.Vulnerabilities.Vulnerability;
 
                var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,
                                function = vulnerability.AnalysisInfo.Unified.Context.Function.name,
                                file = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path,
                                line = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line,
                                col = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.colStart
                               
                            }).toList();
                tableList.title("Showing {0} Vulnerabilties".format(results.size()))
                         .show(results);                                       
                
              };

Next add Drag & Drop support so that can just drop an *.fvdl file to see it:

Action<string> loadAndShowFile =
    (file)=>{ 
                 var fvdl = loadFvdl(file);
                 showFvdl(fvdl);
            };

tableList.onDrop(loadAndShowFile);
tableList.getListViewControl().onDrop(loadAndShowFile);

…and show a message to the user (while loading the data in a separate thread)

Action<string> loadAndShowFile =
    (file)=>{
                tableList.title("... loading file: {0}".format(file.fileName()));
                O2Thread.mtaThread(()=>{
                                            var fvdl = loadFvdl(file);
                                            showFvdl(fvdl);
                                        });
            };
tableList.onDrop(loadAndShowFile);
tableList.getListViewControl().onDrop(loadAndShowFile);

Change the getFvdl method to add support for caching the loaded objects (helps when dealing with large files that are loaded more than one time during the same session)

Func<string, FVDL> loadFvdl =
    (fvdlFile)=>{       
                    try
                    {
                        return (FVDL)O2LiveObjects.get(fvdlFile);
                    }
                    catch { }
                    
                    var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();       
                     var _fvdl = FVDL.Load(fvdlFile);   
                     O2LiveObjects.set(fvdlFile,_fvdl);
                     o2Timer.stop();
                     return _fvdl; 
                };

Change the getFvdl method to detect some cases where there is no data for: function, file or line

Action<FVDL> showFvdl =
     (_fvdl)=>{
                var vulnerabilities = _fvdl.Vulnerabilities.Vulnerability;
 
                var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,                                          
                                function = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.Function.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.Function.name
                                            : "" ,
                                file = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path
                                            : "" ,
                                line = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line
                                            : 0
                            }).toList();
                tableList.title("Showing {0} Vulnerabilties".format(results.size()))
                         .show(results);                                                        
              };

Make this a generic tool and add a title to the TableList that indicates to the user that he/she needs to drop an *.fvdl file to load it:

//var topPanel = panel.clear().add_Panel();
var topPanel = "Util - Simple FVDL viewer".popupWindow(1000,400);</pre>
&nbsp;

var tableList = topPanel.clear().add_TableList().title("Drop an *.fvdl file here to load it");

Finally, save it as an *.h2 file so that it can be invoked as a stand alone tool:

To execute this script, just double click on it, and the following GUI should appear:

Now drag and drop a *.fvdl file to load it and see detals about its vulnerabilities:

 naim.fvdl

lighttpd.fvdl

nagios.fvdl

mvnforum.fvdl

For reference here is the complete script (available as the Util – Simple FVDL viewer.h2 script):

//var topPanel = panel.clear().add_Panel();
var topPanel = "Util - Simple FVDL viewer".popupWindow(1000,400);

var tableList = topPanel.clear().add_TableList().title("Drop an *.fvdl file here to load it");

Func<string, FVDL> loadFvdl =
    (fvdlFile)=>{       
                    try
                    {
                        var chachedFvdl = (FVDL)O2LiveObjects.get(fvdlFile);
                        if (chachedFvdl.notNull())
                            return chachedFvdl;
                    }
                    catch { }
                    
                    var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();       
                     var _fvdl = FVDL.Load(fvdlFile);   
                     O2LiveObjects.set(fvdlFile,_fvdl);
                     o2Timer.stop();
                     return _fvdl; 
                };
 
 Action<FVDL> showFvdl =
     (_fvdl)=>{
                var vulnerabilities = _fvdl.Vulnerabilities.Vulnerability;
 
                var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,                                          
                                function = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.Function.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.Function.name
                                            : "" ,
                                file = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path
                                            : "" ,
                                line = vulnerability.AnalysisInfo.Unified.notNull() && vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.notNull()
                                            ? vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line
                                            : 0
                            }).toList();
                tableList.title("Showing {0} Vulnerabilties".format(results.size()))
                         .show(results);                                                        
              };
 

Action<string> loadAndShowFile =
    (file)=>{
                tableList.title("... loading file: {0}".format(file.fileName()));
                O2Thread.mtaThread(()=>{
                                            var fvdl = loadFvdl(file);
                                            showFvdl(fvdl);
                                        });
            };

tableList.onDrop(loadAndShowFile);
tableList.getListViewControl().onDrop(loadAndShowFile);
   

return "done"; 

//using xmlns.www.fortifysoftware.com.schema.fvdl
//O2File:C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\Fortify.fvdl.1.6.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

Fortify FVDL files – Creating and consuming the schema and CSharp file

An o2 user send me a nice 400Mb Fortify file to see if O2 could do something with it (since Fortify’s GUI was not able to load it).

The next couple posts will document how to go from an XML file to creating an API to consume and to visualize it.

For this demo I will use the Fortify FVDL files that where published by SATE 2008 results (part of NIST SAMATE project) which Fortify participated. You can download the entire SATE 2008 data from their website (which includes the findings files from all participats and SATE’s result), or you can download just the Fortify FVDL (and xsd) from O2’s S3 repository: Fortify-Sate-2008.zip (the scripts below will use this files, but if you have access to *.fvdl files, you can use them)

To start, open an ‘O2 Quick Development Environment GUI’

Then download the demos files and unzip them to a local temp folder:

var demoFileUrl = "http://s3.amazonaws.com/Demo_Files/Fortify-Sate-2008.zip";
var localFile = demoFileUrl.uri().download();

Next step is to unzip the file:

var localFile = "".tempDir().pathCombine("Fortify-Sate-2008.zip");
if (localFile.fileExists().isFalse())
{
    var demoFileUrl = "http://s3.amazonaws.com/Demo_Files/Fortify-Sate-2008.zip";
    demoFileUrl.uri().download();
}   
return localFile.unzip_File();

Then add a check to only unzip if the target folder doesn’t exist and package it in a Lambda method which can be easily consumed by the main script:

Func<string> getFolderWithFvdlDemoFiles =
    ()=>{
            var localFile = "".tempDir().pathCombine("Fortify-Sate-2008.zip");
            if (localFile.fileExists().isFalse())
            {
                var demoFileUrl = "http://s3.amazonaws.com/Demo_Files/Fortify-Sate-2008.zip";
                demoFileUrl.uri().download();
            }    
            var targetFolder = @"..\_Fortify-Sate-2008".tempDir(false).fullPath(); // by default this willl resolve to C:\O2\_tempDir\_Fortify-Sate-2008
            if (targetFolder.dirExists().isFalse())
                localFile.unzip_File(targetFolder);            
            return targetFolder;
        };

var folderWithFvdlFiles = getFolderWithFvdlDemoFiles();       
return folderWithFvdlFiles.files(true,"*.fvdl");

There a number of *.fvdl files available, and since the first thing we need is to create an *.xsd for them, let’s pick the smaller one (in this case naim.fvdl)

var folderWithFvdlFiles = getFolderWithFvdlDemoFiles();        
var naimFvdl = folderWithFvdlFiles.files(true,"naim.fvdl").first();
return naimFvdl;

Usually at this stage we can use the O2 Fluent XML Apis to do this. Basically, in most cases this works:

var folderWithFvdlFiles = getFolderWithFvdlDemoFiles();        
var naimFvdl = folderWithFvdlFiles.files(true,"naim.fvdl").first();


return naimFvdl.xmlCreateXSD(); // creates XSD from Xml file

Unfortunately, for this XML file, we get the error “The table (Node) cannot be the child table to itself in nested relations.” (which is quite a common error when creating XSDs from XML files)

The 2nd usual attempt is to try to use the XSD.exe tool that comes with Visual Studio, But that just returns the same error:

The 3rd attempt is to use VisualStudio 2010 for the conversion. Before we open the file in visual studio, we will need to create a new copy (or rename it) with an .xml extension (ie C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.xml)

 Then from the XML menu, select the ‘Create Schema’ menu Item

Which this time around should work 🙂 :

Save this file locally

Now go back to the O2 development Gui (or open up a new one), and lets create the CSharp file. Start by making sure you have a correct reference to the file:

var xsdFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.xsd";
return xsdFile.fileExists();

call the xsdCreateCSharpFile method to create a CSharp file:

var xsdFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.xsd";
return xsdFile.xsdCreateCSharpFile(); 

This will create a file called ‘C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.cs’, which you can take look by opening it in a source code editor control

var topPanel = panel.clear().add_Panel();
var xsdFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.xsd";
var csharpFile = xsdFile.xsdCreateCSharpFile(); 
var codeEditor = topPanel.add_SourceCodeEditor(); // add source code editor to topPanel
codeEditor.invoke("setMaxLoadSize","1000");       // invoke the private method that sets the max size of the file to load using the SharpDevelop control (if bigger the file is opended in a listview)
codeEditor.open(csharpFile);                        // open file

if try to compile this csharp file, you will get a number of “The type of namespace ‘xyz’ could not be found”  errors

This is caused because the CSharp file was created using the Link2Xml Apis which are included in the O2_Misc_Microsoft_MPL_Libs.dll.

The solution is to add this as a reference at the top of the file, which will make the compilation work

Note that this extra reference could had also be introduced programatically like this:

var xsdFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.xsd";
var csharpFile = xsdFile.xsdCreateCSharpFile(); 

csharpFile.fileContents()
      .insertBefore("//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll".line())
      .saveAs(csharpFile);

The final step, now that we have the CSharp file (‘C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.cs’), is to use it as a reference and load up the original Xml file using it.

To make it clear, lets do this on a clean O2 Development GUI Environment

If you look at the CSHarp file you should notice that there is namespace

namespace xmlns.www.fortifysoftware.com.schema.fvdl {

…which contains a FVDL class

public partial class FVDL : XTypedElement, IXMetaData {

…which contains a Load static method

        public static FVDL Load(string xmlFile) {
            return XTypedServices.Load<FVDL>(xmlFile);
        }

This is the method that we will need to call , so that we can get a strongly typed version of our XML file

var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
return fvdl;

//using xmlns.www.fortifysoftware.com.schema.fvdl
//O2File:C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

 … which finally gives us the object that we want:

…and provides strongly-typed access to the fvdl data, including code-complete support

  ->

As a first exampe,  here is the list of vulnerability objects in the loaded xml file:

Note: one way to see in more details the type of objects that are availble is to call the {object}.details() method:

var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
var vulnerabilities = fvdl.Vulnerabilities.Vulnerability; 
vulnerabilities.details();

..which opens a popup window that can be navigated by properties or fields (note that the data is only loaded on selection)

Going back into Vulnerability filtering …

… where it gets really powerful is when we use Linq queries to filter the data:

var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
var vulnerabilities = fvdl.Vulnerabilities.Vulnerability; 

return (from vulnerability in vulnerabilities
        select vulnerability.ClassInfo.Kingdom).Distinct(); 

In this case, here is a distinct list of the vunerability’s Kingdoms

…  or a quick consolidated view of  the vulnerability data:

var topPanel = panel.clear().add_Panel();
var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
var vulnerabilities = fvdl.Vulnerabilities.Vulnerability; </pre>
&nbsp;

var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,
                                function = vulnerability.AnalysisInfo.Unified.Context.Function.name,
                                file = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path,
                                line = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line,
                                col = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.colStart
                               
                            });
return results;

 

This  ‘Linq result’ object, is a local C# anonymous class, and is better seen and analyed on a TableList:

var topPanel = panel.clear().add_Panel();
var xmlFile = @"C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl";
var fvdl = FVDL.Load(xmlFile);
var vulnerabilities = fvdl.Vulnerabilities.Vulnerability; </pre>
&nbsp;

var results =  (from vulnerability in vulnerabilities
                  select new  {
                                kingdom = vulnerability.ClassInfo.Kingdom,
                                analyzer = vulnerability.ClassInfo.AnalyzerName,
                                classId = vulnerability.ClassInfo.ClassID,
                                defaultSeverity = vulnerability.ClassInfo.DefaultSeverity,
                                instanceId = vulnerability.InstanceInfo.InstanceID,
                                instanceSeverity = vulnerability.InstanceInfo.InstanceSeverity,
                                confidence = vulnerability.InstanceInfo.Confidence,
                                function = vulnerability.AnalysisInfo.Unified.Context.Function.name,
                                file = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.path,
                                line = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.line,
                                col = vulnerability.AnalysisInfo.Unified.Context.FunctionDeclarationSourceLocation.colStart
                               
                            }).toList();

topPanel.add_TableList("Showing {0} Vulnerabilties".format(results.size()))
        .show(results);                           
return "done";

//using xmlns.www.fortifysoftware.com.schema.fvdl
//O2File:C:\O2\_tempDir\_Fortify-Sate-2008\Fortify-Sate-2008\sate2008-Fvdl\naim.fvdl.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll

That wraps up this example, the next related entries will continue from here 🙂

Using ‘Amazon S3 Browser’ script to upload files to Amazon S3

If you have an Amazon S3 account (http://aws.amazon.com/s3/) then you might find useful to use the ‘Amazon S3 Browser.h2’ script to upload files to the S3 servers (you can also create folders and delete files)

To upload files, select the target foldder (top left treeview) and  just drop the desired file on the the bottom left treeview (the red color indicates that the file is being uploaded)

When the upload has completed, the bottom treeview will refresh and show the uploaded file there:

To get the link to the file, click on it and copy it from the top right textbox

For small files (like images or html files) you can get a quick preview by selecting the ‘Auto Preview Selected File’ checkbox

Note that when you upload files using this GUI/Script the file permission will be set to public (the default is to make it private).

The ‘Amazon S3 Browser.h2‘ script (shown below) consumes the API_AmazonS3.cs file and is mainly focused on building the GUI:

//O2Ref:WatiN.Core.1x.dll
var topPanel = O2Gui.open<Panel>("Amazon S3 Browser",700,400);
var autoPreviewSelectedFile = false;
var amazonS3 = new API_AmazonS3();  </pre>
&nbsp;

topPanel.add_Label("Logging in to S3", 150,300);
amazonS3.login();
topPanel.clear();
var statusLabel = topPanel.parentForm().add_StatusStrip(); 
statusLabel.set_Text("Welcome to O2's Amazon S3 Browser (click on folder to see files)");
var foldersTreeView = topPanel.add_TreeView()
                           .showSelection();
                          
var previewArea = foldersTreeView.insert_Right<Panel>(topPanel.width() /2 ).add_1x1("Selected File Url", "Selected File Contents",false, 65) ;
var previewPanel = previewArea[1];
var selectFile = previewArea[0].add_TextBox().fill();
previewArea[0].add_CheckBox("Auto Preview selected File (to see single files just press enter above)",40,4, (value) => autoPreviewSelectedFile = value).autoSize(); 

var filesTreeView = foldersTreeView.insert_Below<TreeView>(topPanel.height()/2)
                                   .showSelection();  

MethodInvoker refreshFolders =
    ()=>{
            foldersTreeView.clear();
            foldersTreeView.add_Nodes(amazonS3.folders());
        };
       
MethodInvoker refreshFileView =
    ()=>{
            amazonS3.showFilesInTreeView(filesTreeView, foldersTreeView.selected().get_Text());           
        };       
                                  
foldersTreeView.insert_Below<Panel>(25)  
               .add_LabelAndTextAndButton("New folder name:", "", "create",
                           (newFolderName)=>
                               {
                                statusLabel.set_Text("Creating folder:{0}".format(newFolderName));
                                   if (amazonS3.create_Folder(newFolderName).valid())
                                   {
                                       foldersTreeView.clear();                                      
                                    foldersTreeView.add_Nodes(amazonS3.folders());   
                                   };
                               });
filesTreeView.onDrop(
    (fileOrFolder)=>{
                        if (fileOrFolder.isFile())
                        {
                            filesTreeView.backColor(Color.LightPink);
                            O2Thread.mtaThread(
                                ()=>{
                                        statusLabel.set_Text("adding file:{0}".format(fileOrFolder));
                                        var currentFolder = foldersTreeView.selected().get_Text();
                                        amazonS3.add_S3_Object(currentFolder, fileOrFolder);
                                        amazonS3.showFilesInTreeView(filesTreeView, currentFolder);                           
                                        statusLabel.set_Text("Done");
                                        filesTreeView.backColor(Color.White);
                                    });
                        }
                           
                    });
  
var foldersContextMenu = foldersTreeView.add_ContextMenu();
var filesContextMenu = filesTreeView.add_ContextMenu();

//menu actions 
foldersContextMenu.add_MenuItem("Delete Folder", 
                            ()=>{
                                    var folderToDelete = foldersTreeView.selected().get_Text();
                                    if ("Are you sure you want to Delete the folder: {0}".format(folderToDelete).askUserQuestion())
                                        if (amazonS3.delete_Folder(folderToDelete).valid())                                      
                                        {
                                            statusLabel.set_Text("Deleting folder:{0}".format(folderToDelete));
                                               refreshFolders();
                                           }
                                });
 
filesContextMenu.add_MenuItem("Delete File", 
                            ()=>{
                                    //filesTreeView.SelectedNode =
                                    var currentFolder = foldersTreeView.selected().get_Text();
                                    var fileToDelete = filesTreeView.selected().get_Text();
                                    if ("Are you sure you want to Delete the file: {0}".format(fileToDelete).askUserQuestion())                                   
                                    {
                                        statusLabel.set_Text("Deleting File:{0}".format(fileToDelete));
                                        if (amazonS3.delete_File(currentFolder, fileToDelete).valid())                                      
                                            amazonS3.showFilesInTreeView(filesTreeView, currentFolder);
                                    }
                                });

// events
foldersTreeView.afterSelect<string>(
    (folder) =>{
                    statusLabel.set_Text("Showing files in folder: {0}".format(folder));
                    refreshFileView();
                });

filesTreeView.afterSelect<Uri>(
    (uri)=>    {               
                var url = uri.str();
                "url: {0}".info(url);
                statusLabel.set_Text("Showing details for file: {0} ".format(url));
                selectFile.set_Text(url);               
                if (autoPreviewSelectedFile)
                    amazonS3.showFileInControl(previewPanel, uri);                                       
            });

selectFile.onEnter(
    (url)=> {               
                amazonS3.showFileInControl(previewPanel, url.uri());       
             });

// ContextMenu

filesContextMenu.add_MenuItem("Refresh", refreshFileView);

var folders = amazonS3.folders();
foldersTreeView.add_Nodes(folders);   
amazonS3.showFilesInTreeView(filesTreeView,"");

return "done";
//using O2.External.IE.ExtensionMethods

//O2File:API_AmazonS3.cs
//O2Ref:AWSSDK.dll
//O2Ref:O2_External_IE.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Xml.Linq.dll

 

Using OpenPGP to Easily create temp PGP keys for secure file exchange

If you need to quickly create PGP keys that you can distribute for temporary use, take a look at the O2 Script ‘tool – using openpgp to encrypt and decrypt.h2’.

This is what the main GUI looks like:

Go to ‘Create or Edit Keys’ and click on ‘Create’ (this can take a couple seconds since OpenPgp needs to get enough entropy from the local system for key generation)

Now just go to the target folder and send the public key file to your receipient.

This Gui can also be used to encrypt text:

Decrypt text:

and Encrypt/Decypt files

Injecting O2 into another .NET Process (in this case NUnit.exe)

Here is a pretty powerful example of what can be done with O2’s .NET reflection APIs.

The objective is to start NUnit under the control of an O2 script and to add a new feature to NUnit (in this case a new error viewer)

There are multiple scripts at play here

1) Inject O2 into – NUnit.h2:  O2 Gui to edit+compile+launch the O2_1st_Script.cs and O2_Launcher
2) O2_Launcher.cs: generic O2 launcher that will look for the O2_1st_Script.cs file, compile it and execute it
3) O2_1st_Script.cs: starts the NUnit gui (WinForms) and once that is started, loads and runs a couple of extra O2 scripts (that have NUnit specific actions)
4) inject error viewer.h2– O2 script that adds the new feature to NUnit
5) Run NUnit – BusinessLogic Class.h2 – O2 script that does a couple actions on the NUnit Gui (in this case select a node from the treeView and click on the ‘Run’ button)

This is what the ‘patched’ version of NUnit looks like:

Here is what the first script (Inject O2 into – NUnit.h2) looks like:

Here is the Source code for each of the relevant scripts

1)  Inject O2 into – NUnit.h2:  O2 Gui to edit+compile+launch the O2_1st_Script.cs and O2_Launcher

</pre>
&nbsp;

var topPanel = O2Gui.open<Panel>("Inject O2 into other apps",700,400);
//var topPanel = panel.clear().add_Panel();
var actionsPanel = topPanel.insert_Above(40);
var codeEditor = topPanel.add_SourceCodeEditor();
topPanel.insert_LogViewer();

var currentFolder = PublicDI.CurrentScript.directoryName();
var compiledDllsFolder = "";
var compiledDllsFolder_TextBox = actionsPanel.add_TextBox(20,"Target Folder","").onTextChange((text)=> compiledDllsFolder= text);
//return currentFolder;
//currentFolder = @"C:\O2\O2Scripts_Database\O2_Core_APIs";

compiledDllsFolder_TextBox.set_Text(@"C:\O2\_XRules_Local\NUnit_Injection\NUnit-2.5.10.11092\NUnit-2.5.10.11092\bin\net-2.0\lib");

&nbsp;

Action<string,string, string> compileToExtension =
    (file, extension,mainClass) =>{
                "Compiling file: {0} ".debug(file);
                var fileToCompile = currentFolder.pathCombine(file + ".cs");
                var compiledDll = compiledDllsFolder.pathCombine(file + extension);
                if (fileToCompile.fileExists().isFalse())
                    "could not find file to compile: {0}".error(fileToCompile); 
                else
                {
                    var assembly = (mainClass.valid())
                                        ? new CompileEngine().compileSourceFiles(new List<string> {fileToCompile}, mainClass)
                                        : new CompileEngine().compileSourceFiles(new List<string> {fileToCompile}, mainClass, System.IO.Path.GetFileNameWithoutExtension(compiledDll));
                    if (assembly.isNull())
                        "no compiled assembly object created for: {0}".error(fileToCompile);
                    else
                    {
                        Files.Copy(assembly.Location, compiledDll);
                        "Copied: {0} to {1}".info(assembly.Location, compiledDll);
                        if (compiledDll.fileExists().isFalse())
                            "compiled file not created in: {0}".error(compiledDll);
                       
                    }
                } 
            };
 
Action<string> compile =
    (file)=> compileToExtension(file,".dll", ""); 
 
 
Action runCompilationProcess =
    ()=>{
            Files.Copy("O2_Kernel.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2_Interfaces.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2_DotNetWrappers.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2_Views_ASCX.dll".assembly().Location, compiledDllsFolder);
            //required for script execution
            Files.Copy("O2_External_SharpDevelop.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2SharpDevelop.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2_API_AST.dll".assembly().Location, compiledDllsFolder);
            Files.Copy("log4net.dll".assembly().Location, compiledDllsFolder);                       
            Files.Copy("White.Core.dll".assembly().Location, compiledDllsFolder);            //Files.Copy(".dll".assembly().Location, compiledDllsFolder);
            Files.Copy("O2_XRules_Database.exe".assembly().Location, compiledDllsFolder);
           
            compileToExtension("O2_Launcher", ".exe", "V2.O2.Platform.Launcher"); 
           
            Files.Copy(currentFolder.pathCombine("O2_1st_Script.cs"), compiledDllsFolder);
        };

actionsPanel.add_Link("Run Compilation Process", 0,0,()=> runCompilationProcess() )
            .append_Link("Execute O2_Launcher.cs", ()=> compiledDllsFolder.pathCombine("O2_Launcher.exe").startProcess())
            .append_Link("Edit O2_Launcher.cs", ()=> codeEditor.open(currentFolder.pathCombine("O2_Launcher.cs")))
            .append_Link("Edit O2_1st_Script.cs", ()=> codeEditor.open(currentFolder.pathCombine("O2_1st_Script.cs")));

codeEditor.open(currentFolder.pathCombine("O2_Launcher.cs"));
//runCompilationProcess();
return "done";

 
2) O2_Launcher.cs: generic O2 launcher that will look for the O2_1st_Script.cs file, compile it and execute it

// This file is part of the OWASP O2 Platform (http://www.owasp.org/index.php/OWASP_O2_Platform) and is released under the Apache 2.0 License (<a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)
//O2Tag_OnlyAddReferencedAssemblies
using System;
using System.Windows.Forms;
using O2.Kernel.ExtensionMethods; 
using O2.Kernel.Objects;   
using O2.DotNetWrappers.DotNet;
using O2.DotNetWrappers.ExtensionMethods;
using O2.Views.ASCX.Ascx.MainGUI;
using O2.Views.ASCX.classes.MainGUI;
 
//O2Ref:O2_Interfaces.dll
//O2Ref:O2_Kernel.dll
//O2Ref:O2_DotNetWrappers.dll
//O2Ref:O2_Views_ASCX.dll
  
namespace V2.O2.Platform
{ 
    static class Launcher
    {
        /// <summary>
        /// The main entry point for the application.
        /// </summary></pre>
&nbsp;

        [STAThread]
        static void Main(string[] args)
        {
            if (Control.ModifierKeys == Keys.Shift)
                showLogViewer().parentForm().width(1000).height(400); 
            var firstScript = "O2_1st_Script.cs";           
            Console.WriteLine("Welcome to the O2 Platform ...");
           
            "Current AppDomain: {0}".info(AppDomain.CurrentDomain.BaseDirectory);
           
            CompileEngine.lsGACExtraReferencesToAdd.Clear();
            var assembly = new CompileEngine().compileSourceFile(firstScript);
            if (assembly.notNull())           
            {
                Console.WriteLine("Executing script {0} from location {1}".info(firstScript, assembly.Location));
                if (assembly.methods().size()>0)
                {
                    assembly.methods()[0].invoke();
                    Console.WriteLine("Invocation complete");
                }
                else
                    Console.WriteLine("Error: there were no methods in compiled assembly");           
            }
            else
                Console.WriteLine("Error: could not find, compile or execute first script ({0})".format(firstScript));
        }               
       
        public static ascx_LogViewer showLogViewer()
        {
            return O2Gui.open<ascx_LogViewer>();
        }              
    }
}

 
3) O2_1st_Script.cs: loads starts the NUnit gui (WinForms) and once that loads runs a couple of extra O2 scripts (that have NUnit specific actions)

// This file is part of the OWASP O2 Platform (http://www.owasp.org/index.php/OWASP_O2_Platform) and is released under the Apache 2.0 License (<a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)
using System;
using System.Windows.Forms;
using O2.Kernel.ExtensionMethods;
using O2.DotNetWrappers.ExtensionMethods;
using O2.DotNetWrappers.DotNet;
using O2.DotNetWrappers.Windows;
using O2.Views.ASCX.Ascx.MainGUI;
using O2.Views.ASCX.classes.MainGUI;</pre>
&nbsp;

using O2.External.SharpDevelop.ExtensionMethods;
//using O2.DotNetWrappers.H2Scripts;
using O2.XRules.Database.Utils;

//O2Ref:O2_Kernel.dll
//O2Ref:O2_Interfaces.dll
//O2Ref:O2_DotNetWrappers.dll
//O2Ref:O2_Views_ASCX.dll
//O2Ref:O2_XRules_Database.exe
//O2Ref:O2_External_SharpDevelop.dll
//O2Ref:O2SharpDevelop.dll
//O2Ref:O2_API_AST.dll
//O2Ref:log4net.dll

//O2Ref:System.dll
//O2Ref:System.Windows.Forms.dll

//O2Ref:System.Drawing.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Core.dll
//O2Ref:System.Data.dll 
//O2Ref:System.Xml.Linq.dll

//O2Ref:nunit-gui-runner.dll

//O2File:Scripts_ExtensionMethods.cs

namespace V2.O2.Platform
{
    public class Launcher
    {
        /// <summary>
        /// The main entry point for the application.
        /// </summary>
       
        public void Main()
        {
            if (Control.ModifierKeys == Keys.Shift)
                showLogViewer();                        
            "This is O2's first Script!!".info();
            "Current AppDomain: {0}".info(AppDomain.CurrentDomain.BaseDirectory);
            O2Thread.staThread(()=>{NUnit.Gui.AppEntry.Main(new string [] {});});
            O2Thread.staThread(
                ()=>{
                        this.sleep(1000);
                        "Trying to get lastFormLoaded".info();
                        var nUnit ="".lastFormLoaded();
                        if (nUnit.isNull())
                        {
                            "Could not get a reference to the main window".error();
                        }
                        else
                        {
                            "Got reference to the main window".info();
                            //var tab =      nUnit.controls<TabControl>(true)[1];     
                            //var scriptEditor = tab.add_Tab("O2Script").add_Script();
                           
                            var scriptEditor = O2Gui.open<Panel>("NUnit Scripts",600,300).add_Script().parentForm().top(600);                       
                           
                            @"E:\_XRules_Local\NUnit_Injection\Scripts\inject error viewer.h2".compile_H2Script().executeFirstMethod();
                            @"E:\_XRules_Local\NUnit_Injection\Scripts\Run NUnit - BusinessLogic Class.h2".compile_H2Script().executeFirstMethod();
                            
                            
                            
                                           }

                    });
        }   
       
        public static ascx_LogViewer showLogViewer()
        {
            return O2Gui.open<ascx_LogViewer>();                       
        }
    }
}

4) inject error viewer.h2– O2 script that adds the new feature to NUnit

var nUnitGui= "".applicationWinForms()[0];</pre>
&nbsp;

nUnitGui.set_Text( NUnit (O2 version)");

var tabControl = nUnitGui.controls<TabControl>(true)[0];
var tabPage = tabControl.controls()[0];
var error_tabPage =  tabPage.controls()[0];

var textBox = error_tabPage.controls<TextBox>(true)[0];

           
//var popupWindow = "errorPage".popupWindow(800,500);
var popupWindow = tabControl.add_Tab("O2 error viewer").add_Panel();

textBox.onTextChange(
    (text)=>{
                var data = from line in textBox.get_Text().split_onLines()
                            where line.valid()
                            select new { location = line.split(" in ")[0].subString(3),  file = line.split(" in ")[1].split(":line")[0], lineNumber =line.split(" in ")[1].split(":line")[1] };

                var popupTableList = popupWindow.clear().add_TableList();
                popupTableList.show(data);
               
               
                var codeViewer = popupTableList.insert_Below().add_SourceCodeViewer();
                popupTableList.afterSelect_get_Row(
                    (row)=>{
                                row.values();
                                var file = row.values()[1];
                                var lineNumber = row.values()[2].toInt();
                                codeViewer.open(file).gotoLine(lineNumber);
                            });
                popupTableList.selectFirst();       
            });

return "ok";
 
//O2File:API_GuiAutomation.cs
//O2Ref:White.Core.dll

//O2Ref:nunit-gui-runner.dll
//O2Ref:nunit.uikit.dll
//O2Ref:O2_Kernel.dll
//O2Ref:O2_Interfaces.dll
//O2Ref:O2_DotNetWrappers.dll
//O2Ref:O2_Views_ASCX.dll
//O2Ref:O2_XRules_Database.exe
//O2Ref:O2_External_SharpDevelop.dll
//O2Ref:O2SharpDevelop.dll
//O2Ref:O2_API_AST.dll
//O2Ref:log4net.dll
//O2Ref:System.Drawing.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Core.dll
//O2Ref:System.Data.dll 
//O2Ref:System.Xml.Linq.dll
//O2Ref:System.Windows.Forms.dll
//O2Ref:System.dll

5) Run NUnit – BusinessLogic Class.h2 – O2 script that does a couple actions on the NUnit Gui (in this case select a node from the treeView and click on the ‘Run’ button) 

var nUnitGui= "".applicationWinForms()[0];</pre>
&nbsp;

nUnitGui.set_Text("NUnit (O2 version)");
var treeView = nUnitGui.controls<TreeView>(true)[1];
var node = treeView.nodes()[0];
treeView.mouse_MoveTo_WinForm();

var guiAutomation = new API_GuiAutomation();
guiAutomation.mouse_MoveBy(-110,-150)
             .mouse_Click();

nUnitGui.button("&Run")
        .mouse_MoveTo_WinForm()
        .click();

return "ok";
 
//O2File:API_GuiAutomation.cs
//O2Ref:White.Core.dll

//O2Ref:nunit-gui-runner.dll
//O2Ref:nunit.uikit.dll
//O2Ref:O2_Kernel.dll
//O2Ref:O2_Interfaces.dll
//O2Ref:O2_DotNetWrappers.dll
//O2Ref:O2_Views_ASCX.dll
//O2Ref:O2_XRules_Database.exe
//O2Ref:O2_External_SharpDevelop.dll
//O2Ref:O2SharpDevelop.dll
//O2Ref:O2_API_AST.dll
//O2Ref:log4net.dll
//O2Ref:System.Drawing.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Core.dll
//O2Ref:System.Data.dll 
//O2Ref:System.Xml.Linq.dll
//O2Ref:System.Windows.Forms.dll
//O2Ref:System.dll

Simple Script.h2

For reference this is the base script that can be executed inside NMap AppDomain

return "O2 Platform script";</pre>
&nbsp;
//var nUnitGui= "".applicationWinForms()[0];
//nUnitGui.set_Text("NUnit (O2 version)");
//var tabControl = nUnitGui.controls<TabControl>(true)[0];

&nbsp;

return "ok";
 
//O2File:API_GuiAutomation.cs
//O2Ref:White.Core.dll

//O2Ref:nunit-gui-runner.dll
//O2Ref:nunit.uikit.dll
//O2Ref:O2_Kernel.dll
//O2Ref:O2_Interfaces.dll
//O2Ref:O2_DotNetWrappers.dll
//O2Ref:O2_Views_ASCX.dll
//O2Ref:O2_XRules_Database.exe
//O2Ref:O2_External_SharpDevelop.dll
//O2Ref:O2SharpDevelop.dll
//O2Ref:O2_API_AST.dll
//O2Ref:log4net.dll
//O2Ref:System.Drawing.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Core.dll
//O2Ref:System.Data.dll 
//O2Ref:System.Xml.Linq.dll
//O2Ref:System.Windows.Forms.dll
//O2Ref:System.dll

Consuming Veracode Findings File(s) using O2

If you are a veracode customer (or have access to a report created by its static/analysis engine), you can use O2 to analyze, filter and extend those findinds.

Note that this first post covers only the viewing part. There is a much more advanced O2 integration with veracode which will be documented later (namely the ability to consumer veracode’s DWR APIs directly, download the Findings Traces data, and to glue them with the findings in the original XML reports)

The current viewers can be accessed via the Veracode (Custom O2).h2 script:

which looks like this:

There are 3 ways you can see the veracode findings and all can be accessed via the Main Gui to view Veracode Findings button (you can also open these viewers individually via the buttons under  the Raw Views section)

By default the Main Gui to view Veracode Findings looks like this

 

To load the files drop them in the area that says ‘DROP XML FILE HERE…’ (you can also drop them on each of the view’s treeview or table list)

Once you drop a file, in the default view (which is the View in SourceCodeViewer) you will be able to see the findings filtered by: Category Name, Type , File or Severity

 

For example here is what the by Category Name looks like:

 

Other View: TableList

Click on the View in TableList link (top left) to see the data in a TableList view (note that this is not the raw Veracode xml data, this is already a normalization view of that data created by Linq queries inside this O2 Script)

 

Other View: TreeView

The other view that is available is a TreeView visualization of the raw Veracode Xml document (this is what it looks like if you open that XML file in a Xml viewer)

 

Other View: StandAlone TreeView

The TreeView view, (shown below when opened as a stand alone form) as support for loading multiple findings files (just drop a folder and all xml/zip Veracode XML  files will be loaded)

… drop a folder in the TreeView

And see multiple findings file in the save location:

Using C# Linq To filter the findings

Here are a couple (C# Extension methods) examples of how to use C# Linq based queries to quickly process the veracode findings file:

 public static class API_Veracode_DetailedXmlFindings_ExtensionMethods_Linq_Queries
    {
        public static List<FlawType> flaws(this API_Veracode_DetailedXmlFindings apiVeracode)
        {
            if(apiVeracode.DetailedReport.isNull())
                return new List<FlawType>();
               
            var flaws = from severity in apiVeracode.DetailedReport.severity
                        from category in severity.category            
                        from cwe in category.cwe
                        from flaw in cwe.staticflaws.flaw                                    
                        select flaw;
            return flaws.toList();
        }
       
        public static List<FlawType> @fixed(this List<FlawType> flaws)
        {
            return (from flaw in flaws
                    where flaw.remediation_status == "Fixed"
                    select flaw).toList();
        }
       
        public static List<FlawType> notFixed(this List<FlawType> flaws)
        {
            return (from flaw in flaws
                    where flaw.remediation_status != "fixed"
                    select flaw).toList();
        }
    }

 

        public static ascx_TableList show_In_TableList(this List<FlawType> flaws , Control control)
        {       
            control.clear();
            var tableList = control.add_TableList();
            Action showData =
                ()=>{
                       
                        var selectedRows =  from flaw in flaws
                                            select new {flaw.severity, flaw.categoryname, flaw.issueid,
                                                        flaw.module, flaw.type, flaw.description, flaw.cweid, 
                                                        flaw.exploitLevel, flaw.categoryid,
                                                        flaw.sourcefile, flaw.line,  flaw.sourcefilepath,
                                                        flaw.scope, flaw.functionprototype, flaw.functionrelativelocation};
                 
       
                        tableList.show(selectedRows);
                        tableList.makeColumnWidthMatchCellWidth();
                    };
            tableList.onDrop(
                (file)=>{
                            var apiVeracode = new API_Veracode_DetailedXmlFindings().load(file);
                            flaws = apiVeracode.flaws();
                            showData();
                        });
            if (flaws.size()>0)
                showData();
            else
                tableList.add_Column("note")
                         .add_Row("drop a Veracode DetailedFindings Xml (or zip) file to view it")
                         .makeColumnWidthMatchCellWidth();
               
            return tableList;
        }

Adding an O2 Menu to Notepad++ and ‘listen’ to menu selections

When working on the TeamMentor 3.0 project, I started using Notepad++ to edit its *.HTML and *.cs files. Naturally, after a while, I wanted to access the TeamMentor development automation capabilities that I had already created using O2, so my solution was to add top-level menu to Notepad++ 🙂

Using O2’s Gui automation APIs (including its new API_InjectMenu_AnotherProcess.cs capability) I was able to add a new menu to Notepad++, which is ‘watched’ by a running instance of O2 (who responds to the selected O2 menu items).

Here is what the menu looks like:

Here is what the O2 Script that ‘listen’ to the commands from Notepad++ looks like:

And here is the source code that makes this possible:

var currentScript = PublicDI.CurrentScript;
var currentDirectory = currentScript.directoryName();
API_GuiAutomation guiAutomation = null;
Window mainWindow = null;
var injectedMenu = new API_InjectMenu_AnotherProcess();
injectedMenu.ProcessToInject = "notepad++";
injectedMenu.InjectedProcess_WindowTitle = "Notepad++ (TeamMentor version)";

var teamMentor = new API_TeamMentor();

var commands = new Dictionary<string,Action>();
Action<string> executeH2Script_TM =
(scriptToExecute)=>
O2Thread.mtaThread(()=> currentDirectory.pathCombine(scriptToExecute).executeH2Script());

Action<string> executeH2Script_O2 =
(scriptToExecute)=>
O2Thread.mtaThread(()=> scriptToExecute.local().executeH2Script());

var lastFileResolved = "";

Func<string> getCurrentFilePath =
()=>{
try
{
if (lastFileResolved.valid() && lastFileResolved.fileName() == mainWindow.tabs()[0].SelectedTab.name())
{
"found previously resolve path whose filename matchs the current selected tab text".info();
return lastFileResolved;
}
}
catch(Exception ex) { "in getCurrentFilePath :{0}".error(ex.Message); }
mainWindow.menu("Edit").mouse();//.click(); // this is a issue/bug here which blocks the execution thread when clicking directly on the UIAutomation element
guiAutomation.mouse_Click();
mainWindow.menu("Edit")
.menu("Copy to Clipboard").mouse();
guiAutomation.mouse_Click();
mainWindow.menu("Edit")
.menu("Copy to Clipboard")
.menu("Current Full File path to Clipboard").mouse();
guiAutomation.mouse_Click();
var filePath = "".clipboardText_Get();
if (filePath.valid())
{
lastFileResolved = filePath;
return filePath;
}
"could not get current filepath".error();
return null;
};

Action<string> compileScript =
(scriptToCompile)=>{
O2Thread.mtaThread(
()=>{
"scriptToCompile: {0}".info(scriptToCompile);
if (scriptToCompile.extension(".h2"))
scriptToCompile.compile_H2Script();
else
{
new CompileEngine().compileSourceFile(scriptToCompile);
}
});
};

Action<string> executeScript =
(scriptToCompile)=>{
O2Thread.mtaThread(
()=>{
if (scriptToCompile.extension(".h2"))
scriptToCompile.compile_H2Script().executeFirstMethod();
else
{
new CompileEngine().compileSourceFile(scriptToCompile).executeFirstMethod();
}
});
};

commands.add( "Compile current script file", ()=> compileScript(getCurrentFilePath()))
.add( "Build TM Web Project (MsBuild)", ()=> teamMentor.website_CSProj().runMsBuild())
.add( "Build TM OnlineStorage (MsBuild)", ()=> teamMentor.webServices_Sln().runMsBuild("/t:Rebuild"))
.add( "Start WebServers", ()=> teamMentor.start_WebService().start_WebSite() )
.add( "Open v3.0 TeamMentor Website (new window)" , ()=> teamMentor.view_3_0_WebSite_InNewWindow())
.add( "Open v2.5 TeamMentor Website (new window)" , ()=> teamMentor.view_WebSite_InNewWindow())
.add( "View TeamMentor ContentViewer (via webservices)", ()=> executeH2Script_TM("Tool - TeamMentor - ContentViewer (via webservices).h2"))
.add( "Editing TM asp.net pages", ()=> executeH2Script_TM(@"Tools\Util - Editing TM asp.net pages .h2"))
.add( "TeamMentor- Custom O2", ()=> executeH2Script_TM("TeamMentor- Custom O2.h2"))
.add( "TeamMentor- IE Automation", ()=> executeH2Script_TM("TeamMentor- IE Automation.h2"))
.add( "Execute current script file", ()=> executeScript(getCurrentFilePath()))
.add( "open current script file in SourceCodeEditor", ()=> getCurrentFilePath().showInCodeEditor())
.add( "Close notepad++ Process" , ()=> injectedMenu.InjectedProcess.stop())
.add( "Write and Execute Unit Tests;", ()=> executeH2Script_O2("Write and Execute Unit Tests.h2"));

injectedMenu.MenuTitle = "O2 Menu (for TeamMentor)";
var initialId = 02000;
foreach(var command in commands)
injectedMenu.MenuCommands.add(initialId++, command.Key);
injectedMenu.CommandSelected =
(id, commandKey)=>{
"[InjectedMenu] Command Received: {0}".debug(commandKey);
O2Thread.mtaThread(()=> commands[commandKey]());;
};
injectedMenu.buildGui();

"now using GuiAutomation".info();
guiAutomation = new API_GuiAutomation(injectedMenu.InjectedProcess);
mainWindow = guiAutomation.window(injectedMenu.InjectedProcess_WindowTitle);

var menu = mainWindow.menu(injectedMenu.MenuTitle)
.mouse();//.click();
guiAutomation.mouse_Click();
return "here";

//this.sleep(1000);
//window.menu("File").mouse().click()
//      .menu("Exit").mouse().click();

//using White.Core.UIItems.WindowItems
//using O2.SecurityInnovation.TeamMentor
//O2File:TeamMentor_Config.cs
//O2File:API_TeamMentor.cs
//O2File:API_InjectMenu_AnotherProcess.cs
//O2File:API_GuiAutomation.cs
//O2Ref:White.Core.dll