O2 Script to get YouTube videos list
Based on the Accessing YouTube via its C# API script , here are two more uses of the YouTube C# API:
Video
Scripts
Create HTML list of videos (used to create this blog post entry)
panel.clear();
var googleDevKey = "AI39si4VZRef7hm9N-RBZ5u-5x5N2lIgyEvv45dGgDxCJjltduhlnF9tTmfWV55tFMUWbnThHybbipiXiEufJueJPpF65lkyjg ";
var settings = new YouTubeRequestSettings("API_YouTube",googleDevKey);
settings.PageSize = 50;
var youTubeRequest = new YouTubeRequest(settings);
Feed<Video> feed = youTubeRequest.GetVideoFeed("DinisCruz");
var webBrowser = panel.add_WebBrowser_Control();
var htmlCode = "<ul>";
var index = 1;
foreach(var entry in feed.Entries)
{
htmlCode += "<li><a href='{1}'>{0}</a> - {2:MMM yy}</li>"
.format(entry.str(),
entry.WatchPage.str(),
entry.AtomEntry.Published,
index++);
}
htmlCode += "</ul>";
webBrowser.open(htmlCode.saveWithExtension(".html"));
return "done";
//using Google.YouTube
//using Google.GData.Client
//using Google.GData.YouTube
//O2Ref:Google.GData.Client.dll
//O2Ref:Google.GData.YouTube.dll
//O2Ref:Google.GData.Extensions.dll
Mini Gui to View Blog Posts
panel.clear();
var googleDevKey = "AI39si4VZRef7hm9N-RBZ5u-5x5N2lIgyEvv45dGgDxCJjltduhlnF9tTmfWV55tFMUWbnThHybbipiXiEufJueJPpF65lkyjg ";
var settings = new YouTubeRequestSettings("API_YouTube",googleDevKey);
var youTubeRequest = new YouTubeRequest(settings);
Feed<Video> feed = youTubeRequest.GetVideoFeed("DinisCruz");
var treeView = panel.clear().add_Panel().add_TreeView();
var webBrowser = treeView.insert_Right().add_WebBrowser_Control();
treeView.afterSelect<Video>(
(video)=>{
webBrowser.open(video.WatchPage.str());
});
foreach(var entry in feed.Entries)
treeView.add_Node(entry);
treeView.selectFirst();
//feed.details();
//panel.add_TableList().show(feed.Entries);
return "done";
//using Google.YouTube
//using Google.GData.Client
//using Google.GData.YouTube
//O2Ref:Google.GData.Client.dll
//O2Ref:Google.GData.YouTube.dll
//O2Ref:Google.GData.Extensions.dll
New version of O2, GitHub based, running in OSX and in VisualStudio
I just wrote a number of blog posts about the new version of O2:
- Installer for O2 Platform v4.0 (first release) – GitHub based
- O2 and REPL – O2 script environment is REPL
- Running O2 REPL Script environment inside Visual Studio 2010 – you can write O2 scripts inside Visual Studio
- Running O2 natively on OSX (via mono) – but not 100% stable – first PoC of running O2 under mono on OSX
O2 Script to create an Amazon EC2 Image
If you want to automate the creation of an EC2 image (for example the O2 Platform one) , here is a script that does that:
Gui Screenshot:
Source Code:
//var topPanel = panel.clear().add_Panel();
var topPanel = "start amazon instance".popupWindow();
var apiKey = @"C:\....\accounts.xml".credential("AmazonAWS");
var amazonEC2 = new API_AmazonEC2(apiKey);
var ec2Client = amazonEC2.getEC2Client();
var width = 400;
var height = 150;
var createPanel = topPanel.add_GroupBox("Create amazon EC2 Instance")
.fill(false)
.width(width).height(height)
.add_Panel();
var imageID = createPanel.add_Label("Image Id:",5,0)
.append_Control<ComboBox>()
.width(200)
.top(0).left(90);
imageID.append_Link("View Image's List",
()=>amazonEC2.show_ImagesList_In_TreeView(ec2Client))
.top(5);
var imageSize =createPanel.add_Label("Image Size:",30,0)
.append_Control<ComboBox>()
.dropDownList()
.top(25).left(90);
var securityGroup = createPanel.add_TextBox(60,"Security Group","").left(90).width(120);
var keyName = createPanel.add_TextBox(80,"Pem Key name","").left(90).width(120);
var defaultImages = new Dictionary<string,string>();
defaultImages.add("O2 Platform Image", "OWASP O2 Platform - Windows 2008")
.add("Amazon Win 2008 32bit IIS+SQL", "Windows-Server2008-i386-SqlExpress-v103")
.add("Amazon Win 2008 64bit IIS+SQL", "Windows-2008R2-SP1-English-SQLExpress-v101")
.add("Amazon Win 2008 64bit IIS+SQL", "Windows-Server2003R2-i386-SqlExpress-v109")
.add("Microsoft Win 2008 64bit IIS+SQL", "Optimized-SQL-Server-2008R2-3-2-11");
imageID.add_Items(defaultImages.keys());
imageSize.add_Items(new List<string> {"m1.small", "t1.micro", "c1.medium","m1.large", "m1.xlarge"});
imageID.select_Item(0);
imageSize.select_Item(0);
securityGroup.set_Text("O2 Boxes");
keyName.set_Text("O2 Key");
var statusMessage = createPanel.add_Label("...", 90,230);
createPanel.add_Button("Launch Image", 60,230)
.align_Right(createPanel).widthAdd(-10)
.onClick(()=>{
statusMessage.set_Text("...launching image").textColor(Color.DarkBlue);
try
{
var imageId = imageID.get_Text();
if (defaultImages.hasKey(imageId))
imageId = amazonEC2.getImagesList(ec2Client).name(defaultImages[imageId]).ImageId;
"using imageId: {0}".info(imageId);
var runningInstance = ec2Client.launchInstance(imageId,
imageSize.get_Text(),
securityGroup.get_Text(),
keyName.get_Text());
statusMessage.set_Text("done, created instance: {0}".format(runningInstance.InstanceId)).textColor(Color.DarkGreen);
}
catch(Exception ex)
{
statusMessage.set_Text(ex.Message).textColor(Color.Red);
ex.log();
}
});
return "done";
//here are a number of code snippets that show the multiple methods available in the API_AmazonEC2
//using Amazon.EC2.Model
//using Amazon.EC2
//return images.description("WebMatrix Hosting Server provided by Microsoft");
//return images.description_Contains("Amazon Linux");
//return images.amazon().description_RegEx("windows.*32");
//return images.name_Contains("amzn-ami");
//return images.name_RegEx("amzn.*i386");
//return images.imageId(".....");
//return images.imageId("ami-9d1043d8");
//return <a href="mailto:images.@public().amazon().windows">images.@public().amazon().windows</a>();
//return <a href="mailto:images.@public().amazon()._32Bit">images.@public().amazon()._32Bit</a>();
//return <a href="mailto:images.@public().amazon()._64Bit">images.@public().amazon()._64Bit</a>();
//return <a href="mailto:images.@public().amazon">images.@public().amazon</a>();
//return <a href="mailto:images.@public().microsoft">images.@public().microsoft</a>();
//return <a href="mailto:images.@public()._32Bit">images.@public()._32Bit</a>();
//return <a href="mailto:images.@public()._64Bit">images.@public()._64Bit</a>();
//return <a href="mailto:images.@private">images.@private</a>();
//return <a href="mailto:images.@private">images.@private</a>();
//amazonEC2.show_ImagesList_In_TreeView(ec2Client,topPanel);
return "ok";
//O2File:API_AmazonEC2.cs
//O2Ref:AWSSDK.dll
O2 Platform EC2 Image
Note: if you want to use the Amazon Web Interface to start the O2 AMI, here is how to find it (NOTE that this is in US West):
Injecting O2 into IBM Rational AppScan Standard
If you use AppScan Standard (the BlackBox tool) and want to use O2 to script its GUI or scans, here is a new O2 script that will start AppScan Standard under the same process as O2.
Once the main AppScan gui is loaded this script will add a new O2 Menu and inject an O2 Script editor.
You can find the script that automates the process in your local folder C:\O2\O2Scripts_Database\_Scripts\3rdParty_Tools\IBM\AppScan_Standard:
The script you want to execute is the Util – Launch AppScan Standard (O2 version).h2 which will
- Copy all required files (O2 dlls and AppScan_Standard_1st_Script.cs Script) to the AppScan Folder.
- Compile the AppScan_Standard_O2_Launcher.cs script into AppScan_Standard_O2_Launcher.exe and copy it to AppScan folder
- In the AppScan folder: execute and compile AppScan_Standard_O2_Launcher.exe
- The AppScan_Standard_O2_Launcher.exe will compile the AppScan_Standard_1st_Script.cs and execute it
- the AppScan_Standard_1st_Script.cs script will:
- open an O2 Log Viewer (so that you get a feel for what is happening),
- launch AppScan (by using reflection to open the MainForm Form control),
- wait for the main AppScan gui to load, and when it does:
- compile and execute the In AppScan – Create O2 Gui.h2 script
- The In AppScan – Create O2 Gui.h2 script is the one that adds a new menu item and injects a O2 Script editor into the main AppScanGui.
The Util – Launch AppScan Standard (O2 version).h2 has an editor for all these scripts, but by default this is disabled (see NUnit example for details). The default behaviour of Util – Launch AppScan Standard (O2 version).h2 (when double-clicked from windows explorer) is to execute the above steps in sequence. And if all goes according to plan, you will see the following ‘AppScan Standard – O2 Version’ GUI:
… note the extra O2 Menu and the O2 Scripting Environment….
For reference, here is what it looks like while AppScan Standard is loading up (the cmd.exe window was created by AppScan_Standard_O2_Launcher.exe and the LogViewer shows the wait for the main AppScan GUI to load up)
These series of scripts are heavily based on the ones previously used and documented in the Injecting O2 into another .NET Process (in this case NUnit.exe) script (see that blog post for more technical details about how this works).
O2 Community Script: Get messages from OWASP mailman
Daniel (from the OWASP .NET Project) sent me the O2 script below which is designed to consume the data exposed by the OWASP Mailman (mailing list manager).
This is the first pass at the creation of a tool that is able to:
- grab the data from mailman,
- visualize it,
- parse it, and eventually
- export it into other mailing lists solutions (for example Google Groups)
There is still quite a lot to do here, so if you want to help, go to the https://github.com/o2platform/Scripts-by-O2-Users GitHub repository and create a fork of it.
This is what this script looks like for the Owasp-o2-platform mailing list:
This is what it looks like for the OWASP-NYNJMetro mailing list
This is the script’s source code:
</pre>
#region config
var owaspMailmanUrl = "https://lists.owasp.org/mailman/listinfo";
var owaspArchive = "https://lists.owasp.org/pipermail/";
var owaspListInfo = "https://lists.owasp.org/mailman/listinfo/";
var o2platform = "Owasp-o2-platform";
#endregion
//var topPanel = panel.clear().add_Panel();
var topPanel = O2Gui.open<Panel>("OWASP Mailman Helper", 1200, 900);
var optionsPanel = topPanel.insert_Above<GroupBox>(100).set_Text("Options").add_Panel();
var mainPanel = topPanel.add<GroupBox>().set_Text("").add_Panel();
var threadsPanel = mainPanel.insert_Left<GroupBox>().set_Text("Threads").add_Panel();
var threadBrowser = mainPanel.insert_Left<GroupBox>().set_Text("Thread").add_Browser();
var threadTree = threadsPanel.add_TreeView();
var statsPanel = mainPanel.insert_Left<GroupBox>(400).set_Text("Statistics").add_Panel();//
var ggroupsPanel = optionsPanel.insert_Right<GroupBox>(200).set_Text("Port to google group").add_Panel();
var statPanel = optionsPanel.insert_Right<GroupBox>(500).set_Text("Mailman Group statistics").add_Panel();
Action<string> selectStastType = (list) => {list.info();};
var statsControls = statPanel.add_LabelAndComboBoxAndButton("Select statistics type: ", "", "Draw", selectStastType);
var drawStatsBtn = statsControls.controls<Button>();
var drawStatsCombo = statsControls.controls<ComboBox>();
drawStatsCombo.add_Item("Posts per year");
var statusStrip = topPanel.parentForm().add_StatusStrip(false);
//options panel
var currentList_Label = optionsPanel.insert_Below<Label>().set_Text("Selected mailing list: ...");
Action<string> selectList = (list) => {currentList_Label.set_Text("Selected mailing list: "+list);};
var tab = optionsPanel.add_LabelAndComboBoxAndButton("Select list: ", "", "Load", selectList);
var list_CombBox = tab.controls<ComboBox>();
var loadThreadsBtn = tab.controls<Button>();
var analyseAllBtn = optionsPanel.add_Button(20,20,"Analyse all lists");
var listLinks = new List<HtmlAgilityPack.HtmlNode>();
//chart control
var chart = new Chart();
chart.Dock = DockStyle.Fill;
statsPanel.add(chart);
//threads
//TODO: change from hashtable to proper class
var threads = new List<System.Collections.Hashtable>();
//reading owasp mailing lists
O2Thread.mtaThread(
()=>{
lock(listLinks) {
statusStrip.set_Text(" Loading OWASP mailman lists from OWASP...");
var htmlDocument = owaspMailmanUrl.uri().getHtml().htmlDocument();
listLinks = htmlDocument.select("//table")[0]
.SelectNodes(".//a")
.toList<HtmlAgilityPack.HtmlNode>()
.Where(x=>x.innerHtml().Contains("<strong>")).ToList();
int i = 0;
foreach(var link in listLinks) {
var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
var item = list_CombBox.add_Item(name);
if(name.eq(o2platform))
list_CombBox.select_Item(i);
i++;
}
statusStrip.set_Text(" OWASP mailman lists loaded!");
}
});
loadThreadsBtn.onClick(
() => {
lock(listLinks) {
var selectedList = list_CombBox.get_Text();
foreach(var link in listLinks)
{
var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
if(name == selectedList) {
var nLink = link.attribute("href").value().replace(owaspListInfo, owaspArchive);
O2Thread.mtaThread(
()=>{
threads = new List<System.Collections.Hashtable>();
statusStrip.set_Text(" Downloading threads from "+nLink);
var htmlDocument = nLink.uri().getHtml().htmlDocument();
var threadedLinks = htmlDocument.select("//table")[0]
.SelectNodes(".//a")
.toList<HtmlAgilityPack.HtmlNode>()
.Where(x=>x.Attributes["href"].Value.Contains("thread.html")).ToList();
threadTree.clear();
foreach(var threadedLink in threadedLinks) {
var nnLink = (nLink+"/"+threadedLink.Attributes["href"].Value);
var html = nnLink.uri().getHtml().htmlDocument();
var threadLinks = html.select("//body")[0].SelectNodes(".//a").toList<HtmlAgilityPack.HtmlNode>().Where(x=>x.Attributes["href"].notNull());
foreach(var threadLink in threadLinks) {
if(threadLink.Attributes["href"].Value.regEx("[0-9]{6}\\.html")) {
var threadHtml = nnLink.replace("thread.html",threadLink.Attributes["href"].Value).url().getHtml().htmlDocument();
var h = new System.Collections.Hashtable();
h["Topic"] = threadHtml.select("//h1")[0].InnerHtml;
h["Author"] = threadHtml.select("//b")[0].InnerHtml;
var timeStr = threadHtml.select("//i")[0].InnerHtml;
h["Content"] = threadHtml.select("//pre")[0].InnerHtml;
h["Time"] = DateTime.ParseExact(timeStr.Replace(" EST", "-5:00").Replace(" EDT","-4:00").Replace(" ", " "),"ddd MMM d HH:mm:sszzz yyyy", System.Globalization.CultureInfo.InvariantCulture);
threads.Add(h);
var node = threadTree.add_Node(h["Topic"]);
node.add_Node("Time: {0}".format(h["Time"]));
node.add_Node("Author: {0}".format(h["Author"]));
node.set_Tag(h["Content"]);
}
}
}
statusStrip.set_Text("Threads downloaded from "+nLink);
});
}
}
}
});
analyseAllBtn.onClick(
() => {
threadTree.clear();
foreach(var link in listLinks) {
var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
var nLink = link.attribute("href").value().replace(owaspListInfo, owaspArchive);
var node = threadTree.add_Node(name);
node.add_Node("Link: {0}".format(nLink));
var htmlDocument = nLink.uri().getHtml().htmlDocument();
var trs = htmlDocument.select("//table")[0]
.SelectNodes(".//tr")
.toList<HtmlAgilityPack.HtmlNode>().ToList();
var first = "";
var last = "";
foreach(var tr in trs) {
var td = tr.SelectNodes(".//td")[0];
if(first=="")
first = td.InnerText;
last = td.InnerText;
td.InnerText.log();
}
node.add_Node("First post: {0}".format(first));
node.add_Node("Last post: {0}".format(last));
}
}
);
threadTree.afterSelect(
(treeNode)=> {
if(treeNode.get_Tag().notNull())
threadBrowser.set_Text("<h2>{1}</h2><h3>{2}</h3> <h3>{3}</h3><hr /><br />{0}".format(treeNode.get_Tag().str(), treeNode.get_Text(), treeNode.Nodes[0].get_Text(), treeNode.Nodes[1].get_Text()));
});
drawStatsBtn.onClick(
() => {
statusStrip.set_Text(" Drawing new chart...");
var dict = new Dictionary<int, int>();
if(threads.count()==0) {
statusStrip.set_Text(" No threads for statistics...");
return;
}
foreach(var de in threads) {
var time = (System.DateTime)de["Time"];
if(!dict.Keys.Contains(time.Year))
dict[time.Year] = 0;
dict[time.Year]++;
}
chart.Series.Clear();
foreach(var k in dict.Keys) {
var serie = new Series(k.str());
serie.ChartType = SeriesChartType.Bar;
serie.Points.AddXY(k,dict[k]);
serie.IsValueShownAsLabel = true;
chart.Series.Add(serie);
}
chart.ChartAreas.Add("Chart");
statusStrip.set_Text(" Drawing finished...");
}
);
return "ok";
//using O2.XRules.Database.Utils.ExtensionMethods
//using O2.External.IE.ExtensionMethods
//using System.Windows.Forms.DataVisualization.Charting
//O2File:HtmlAgilityPack_ExtensionMethods.cs
//O2File:IE_JQuery.cs
//O2Ref:WatiN.Core.1x.dll
//O2Ref:O2_External_IE.dll
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Xml.Linq.dll
//O2Ref:System.Windows.Forms.DataVisualization.dll
What ‘Graph’ support exists in O2 today
I got this question today, and nothing better than answer it with some screenshots 🙂
Open the ‘Util – O2 Available scripts’ and do a search for ‘Graph’
ascx_GraphWithInspector.cs.o2
This is a very powerful development gui for the GraphSharp WPF control which is an amazing ‘Real-Time’ Graph visualization engine:
…comment the 2nd line to see a bigger script in action
The showAllLayouts line will show an animation of the diferent types of layout possible:
O2 Xaml Editor (PoC).h2
this is a tool to write and view XAML controls:
Util – Diagram Designer Editor.h2
Do a search for ‘Diagram’ and execute the Util – DiagramDesigner Editor script
This will open a GUI that is a mix of a Diagram Designer and Image Editor (based on the DesignerControl described in this codeproject article)
To see a couple examples of the type of images that can be created using this control look at the example files in the C:\O2\O2Scripts_Database\_Scripts\_DataFiles\SampleDiagrams folder
For example: O2_Videos-splashScreen_WebGoat.xml
Creating PDFs
There is also a really powerful API to manipulate PDFs.
See this post for an example: O2 Script – Creating PDFs with OWASP AppSec Brazil Certificates
Other Graph controls/engines
As with any O2 script, you can easy import and access to any other .NET WinForms or WPF Graph controls, for example the 3.5 Chart Controls distributed by Microsoft.
Submit file to Veracode Trial: using Browser and Windows Automation (WatiN and White APIs)
This script automates the process off submitting a file to Veracode’s free trial (as of 20/Jul/2011). See also Consuming Veracode Findings File(s) using O2.
If first provides a settings GUI, where the user can enter the requried data (email and file to upload), then fires up a web brower and uses WatiN (Browser Automation) plus White (Windows UIAutomation) to populate the form fields and to submit the form.
The reason why I had to use UIAutomation (and White API) was because there didn’t seem to be a way to modify the HTML form field of the ‘file upload’ control (even jQuery didn’t seem to be able to modify that value programatially).
The solution was to use UIAutomation to:
- click on the ‘Browse’ field,
- then (on the popup file dialog window)
- enter the file to upload, and
- click ‘Close’
Here is the ‘settings window:
When the button is clicked the veracode trial page is opened and the ‘Html Form File Upload Button’ is pressed (via UIAutomation) :
When the ‘Choose File to Upload’ window appears, the ‘File Name Text Box’ is populated and the Open Button is pressed (both using UIAutomation (via White API)).
Finally, the email field is populated, the check box is ticked and the ‘Upload’ button is pressed (using Browser Automation (WatiN))
Here is the source code of this script:
var topPanel = O2Gui.open<Panel>("Util - Submit file to Veracode Trial",700,180);
//var topPanel = panel.clear().add_Panel();
topPanel.insert_LogViewer();
var _email = "o2@o2platform.com";
var _fileToUpload = @"C:\O2\Demos\jPetStore - O2 Demo Pack\apache-tomcat-7.0.16\webapps\jpetstore.war";
Action<string,string> submitFileToVeracode =
(email, fileToUpload)=>
{
var windowName= "Veracode File Upload - {0}".format(10.randomLetters());
var ie = windowName.popupWindow(1000,500)
.add_IE().open("https://trial.veracode.com/freetrials/veracode-free-trial-signup.php");
var processId = Processes.getCurrentProcessID();
var apiGuiAutomation = new API_GuiAutomation(processId);
var window = apiGuiAutomation.window(windowName);
"got main window".info();
var buttons = window.buttons();a
"found {0} buttons".info(buttons.size());
buttons[1].mouse().click();
"clicked button".debug();
var selectFileWindow = apiGuiAutomation.window("Choose File to Upload");
selectFileWindow.textBoxes()[0].set_Text(fileToUpload);
selectFileWindow.button("Open").click();
ie.field("email",email);
ie.checkBoxes()[0].check();
ie.button("Upload").click();
//buttons[2].mouse().click();
};
topPanel.add_TextBox("Email",_email).top(0)
.width(100)
.onTextChange((text)=>_email=text)
.append_Label("File to upload")
.autoSize()
.top(3)
.append_TextBox(_fileToUpload)
.onTextChange((text)=>_fileToUpload=text)
.width(300)
.align_Right(topPanel);
topPanel.add_Button(24,"Create Account and Upload File")
.font_bold()
.align_Right(topPanel)
.onClick(()=> submitFileToVeracode(_email, _fileToUpload) );
//O2File:WatiN_IE_ExtensionMethods.cs
//using O2.XRules.Database.Utils.O2
//O2Ref:WatiN.Core.1x.dll
//O2File:API_GuiAutomation.cs
//O2Ref:White.Core.dll
O2 Platform Amazon EC2 Image (AMI)
If you want to take O2 for a test drive and don’t have a windows box (or VM) at hand, now you can use Amazon EC2.
Here are the steps to take:
- Login to Amazon AWS account,
- Go to EC2 (if needed choose the desired region)
- Go to the Instances Section
- Click on ‘Launch Instance
- Search for O2 (the full name of this AMI is ‘727618846505/OWASP O2 Platform – Windows 2008’)
- Click Select (and follow the wizard till the end)
After a couple minutes, you will be able to RDP into your VM.
The default administrator password is: mK-jv7sj@L
After login, start O2 so that it updates the rules, and you are all set 🙂
Let me know if you have any issues using this Amazon EC2 image
Fortify FVDL Files – Simple Viewer based on PropertyGrid
The .NET PropertyGrid Control can be a very powerful way to visualize objects.
For exampe, the script below creates a WinForm with just a PropertyGrid Control (and some drag and drop support to load up the files)
var topPanel = "Util - FVDL viewer (just PropertyGrid) - drop file to load".popupWindow(400, 400); topPanel.insert_LogViewer(); var propertyGrid = topPanel.add_PropertyGrid().helpVisible(false); Action<string> loadAndShowFile = (file)=> O2Thread.mtaThread( ()=> propertyGrid.show(new API_Fortify().convertToFortifyScan(file))); propertyGrid.onDrop(loadAndShowFile); //O2File:API_Fortify_1_6.cs //O2Ref:O2_Misc_Microsoft_MPL_Libs.dll
which looks like this when opened
…and like this after dropping the 36Mb dspace.fvdl file:
Since most Fortify_* objects have been designed as lists, the PropertyGrid will show them in special pop-up windows:
CalledWithNoDefs
Contexts
Descriptions
Scanned Files
Sinks
Snippets
Sources
Vulnerabilities
Vulnerabilities -> Traces
Fortify FVDL Files – Looking at the API_Fortify classes that parse the fvdl data
Here is the source code that create the environment/GUI/tool shown in the Fortify FVDL Files – First working Parser and Viewer for *.fvdl files post
Heavily based on the first version of this API , the data classes have been refactored and expanded, now covering most of the data contained in the *.fvdl files:
public class Fortify_Scan
{
public FVDL _fvdl;
public string BuildID { get; set; }
public uint Loc { get; set; }
public string SourceBasePath { get; set; }
public uint ScanTime { get; set; }
public DateTime CreatedDateTime { get; set; }
public string Errors { get; set; }
public List<Fortify_ScannedFile> ScannedFiles { get; set; }
public List<Fortify_Context> Contexts { get; set; }
public List<Fortify_Description> Descriptions { get; set; }
public List<Fortify_CalledWithNoDef> CalledWithNoDefs { get; set; }
public List<Fortify_Sink> Sinks { get; set; }
public List<Fortify_Source> Sources { get; set; }
public List<Fortify_Snippet> Snippets { get; set; }
public List<Fortify_Vulnerability> Vulnerabilities { get; set; }
public Fortify_Scan()
{
ScannedFiles = new List<Fortify_ScannedFile>();
Contexts = new List<Fortify_Context>();
Descriptions = new List<Fortify_Description>();
CalledWithNoDefs = new List<Fortify_CalledWithNoDef>();
Sinks = new List<Fortify_Sink>();
Sources = new List<Fortify_Source>();
Snippets = new List<Fortify_Snippet>();
Vulnerabilities = new List<Fortify_Vulnerability>();
}
}
public class Fortify_ScannedFile
{
public uint Loc { get; set; }
public uint Size { get; set; }
public ulong Timestamp { get; set; }
public string Type { get; set; }
public string Path { get; set; }
}
public class Fortify_Context
{
public string Id { get; set; }
public Fortify_Function Function { get; set; }
}
public class Fortify_Function
{
public string FunctionName { get; set; }
public Fortify_CodeLocation CodeLocation { get; set; }
public Fortify_Function()
{}
public Fortify_Function(string functionName, object location)
{
FunctionName = functionName;
CodeLocation = new Fortify_CodeLocation(location);
}
public override string ToString()
{
return "{{ name: {0} , path: {1} , line: {2}, lineEnd: {3}, colStart: {4}, colEnd: {5} }}"
.format(FunctionName, CodeLocation.Path, CodeLocation.Line, CodeLocation.LineEnd, CodeLocation.ColStart, CodeLocation.ColEnd);
}
}
public class Fortify_CodeLocation
{
public string Path { get; set; }
public uint Line { get; set; }
public uint LineEnd { get; set; }
public uint ColStart { get; set; }
public uint ColEnd { get; set; }
public Fortify_CodeLocation()
{}
public Fortify_CodeLocation(object location)
{
Path = (string)location.prop("path");
Line = (uint)location.prop("line");
LineEnd = (uint)location.prop("lineEnd");
ColStart = UInt16.Parse(location.prop("colStart").str());
ColEnd = UInt16.Parse(location.prop("colEnd").str());
}
public override string ToString()
{
if (ColStart == 0 && ColEnd==0)
return "{{ path: {0} , line: {1}, lineEnd: {2}}}".format(Path, Line, LineEnd);
return "{{ path: {0} , line: {1}, lineEnd: {2}, colStart: {3}, colEnd: {4} }}"
.format(Path, Line, LineEnd, ColStart, ColEnd);
}
}
public class Fortify_Description
{
public string Abstract { get; set; }
public string ClassID { get; set; }
public string ContentType { get; set; }
public string Explanation { get; set; }
public string Recommendations { get; set; }
public List<string> Tips { get; set; }
public string Tips_as_String { get { return Tips.join(" , ");} } //viewer helper
public Fortify_Description()
{
Tips = new List<string>();
}
}
public class Fortify_CalledWithNoDef
{
public string Name { get; set; }
public string Namespace { get; set; }
public string EnclosingClass { get; set; }
}
public class Fortify_Sink
{
public string RuleID { get; set; }
public Fortify_Function Function_Call{ get; set;}
}
public class Fortify_Source
{
public string RuleID { get; set; }
public Fortify_Function Function_Call{ get; set;}
public Fortify_Function Function_Entry{ get; set;}
public List<string> TaintFlags { get; set;}
public string TaintFlags_as_String { get { return TaintFlags.join(" , ");} } //viewer helper
public Fortify_Source()
{
TaintFlags = new List<string>();
}
}
public class Fortify_Snippet
{
public string Id { get; set; }
public Fortify_CodeLocation CodeLocation { get; set; }
public string Text { get; set;}
}
public class Fortify_ReplacementDefinitions
{
public Items Definitions { get; set; }
//public List<Fortify_Function> LocationDefinitions { get; set; } // add later if needed (Fortify_Function is not the best since the LocationDef has a 'key' value
public Fortify_ReplacementDefinitions()
{
Definitions = new Items();
}
public override string ToString()
{
return "{0} definitions".format(Definitions.size());
}
}
public class Fortify_TraceEntry
{
public uint NodeRefId { get; set; } // when this one is set, the ones below are not set
public string ActionType { get; set; }
public string ActionValue { get; set; }
public bool DetailsOnly { get; set; }
public bool IsDefault { get; set; }
public List<Fortify_TraceEntryFact> KnowledgeFacts { get; set; }
public string Reason_RuleId { get; set; }
public string Reason_TraceRef { get; set; }
public string Reason_Internal { get; set; }
public Fortify_CodeLocation SourceLocation { get; set; }
public Fortify_CodeLocation SecundaryLocation { get; set; }
public uint SourceLocation_ContextId { get; set; }
public string SourceLocation_Snippet { get; set; }
public string SecundaryLocation_Snippet { get; set; }
public string Label { get; set; }
public Fortify_TraceEntry()
{
KnowledgeFacts = new List<Fortify_TraceEntryFact>();
}
}
public class Fortify_TraceEntryFact
{
public bool Primary {get;set;}
public string Type {get;set;}
public string Value {get;set;}
}
public class Fortify_Vulnerability
{
public Fortify_Function Context { get; set; } // from AnalysisInfo
public Fortify_ReplacementDefinitions ReplacementDefinitions { get; set; }
public List<Fortify_TraceEntry> Traces { get; set; }
public string Kingdom { get; set; } // from ClassInfo
public string AnalyzerName { get; set; }
public string ClassId { get; set; }
public decimal DefaultSeverity { get; set; }
public string Type { get; set; }
public string SubType { get; set; }
public decimal Confidence { get; set; } // from InstanceInfo
public string InstanceId { get; set; }
public decimal InstanceSeverity { get; set; }
public Fortify_Vulnerability()
{
ReplacementDefinitions = new Fortify_ReplacementDefinitions();
Traces = new List<Fortify_TraceEntry>();
}
}
The main API_Fortify class is still quite small:
public class API_Fortify
{
public Fortify_Scan convertToFortifyScan(string fvdlFile)
{
var scan = new Fortify_Scan();
scan._fvdl = loadFvdl_Raw(fvdlFile);
scan.mapFvdlData();
return scan;
}
public FVDL loadFvdl_Raw(string fvdlFile)
{
"Loading fvdl file: {0}".info(fvdlFile);
try
{
var chachedFvdl = (FVDL)O2LiveObjects.get(fvdlFile);
if (chachedFvdl.notNull())
return chachedFvdl;
}
catch { }
var o2Timer = new O2Timer("loading {0} file".format(fvdlFile.fileName())).start();
var _fvdl = FVDL.Load(fvdlFile);
O2LiveObjects.set(fvdlFile,_fvdl);
o2Timer.stop();
return _fvdl;
}
}
With all the heavy lifting of convering the *.fvdl data into the multiple O2 Fortify_* classes , done via ExtensionMethods
public static class Fortify_Scan_ExtensionMethods_MappingFvdl
{
public static Fortify_Scan mapFvdlData(this Fortify_Scan fortifyScan)
{
var o2Timer = new O2Timer("Mapped Fvdl Data").start();
fortifyScan.mapScanDetails()
.mapContextPool()
.mapDescriptions()
.mapCalledWithNoDefs()
.mapSinks()
.mapSources()
.mapSnippets()
.mapVulnerabilities();
o2Timer.stop();
return fortifyScan;
}
public static Fortify_Scan mapScanDetails(this Fortify_Scan fortifyScan)
{
var fvdl = fortifyScan._fvdl;
fortifyScan.BuildID = fvdl.Build.BuildID;
fortifyScan.Loc = fvdl.Build.LOC;
fortifyScan.SourceBasePath = fvdl.Build.SourceBasePath;
fortifyScan.ScanTime = fvdl.Build.ScanTime.value;
fortifyScan.CreatedDateTime = DateTime.Parse("{0} {1}".format(fortifyScan._fvdl.CreatedTS.date.ToShortDateString() ,
fortifyScan._fvdl.CreatedTS.time.ToLongTimeString() ));
fortifyScan.Errors = fvdl.EngineData.Errors.str()!= "<Errors xmlns=\"xmlns://www.fortifysoftware.com/schema/fvdl\" />"
? fvdl.EngineData.Errors.str()
: "";
foreach(var file in fvdl.Build.SourceFiles.File)
fortifyScan.ScannedFiles.Add(new Fortify_ScannedFile ()
{
Loc = file.loc,
Size = file.size,
Timestamp = file.timestamp,
Type = file.type,
Path = file.TypedValue
});
return fortifyScan;
}
public static Fortify_Scan mapContextPool(this Fortify_Scan fortifyScan)
{
foreach(var context in fortifyScan._fvdl.ContextPool.Context)
{
//var location = context.FunctionDeclarationSourceLocation;
fortifyScan.Contexts.Add(
new Fortify_Context()
{
Id = context.id.str() ,
Function = new Fortify_Function(context.Function.name,context.FunctionDeclarationSourceLocation)
});
}
return fortifyScan;
}
public static Fortify_Scan mapDescriptions(this Fortify_Scan fortifyScan)
{
foreach(var description in fortifyScan._fvdl.Description)
{
var fortifyDescription = new Fortify_Description
{
Abstract = description.Abstract,
ClassID = description.classID,
ContentType = description.contentType,
Explanation = description.Explanation,
Recommendations = description.Recommendations,
};
if (description.Tips.notNull())
foreach(var tip in description.Tips.Tip)
fortifyDescription.Tips.Add(tip);
fortifyScan.Descriptions.add(fortifyDescription);
}
return fortifyScan;
}
public static Fortify_Scan mapCalledWithNoDefs(this Fortify_Scan fortifyScan)
{
//have to map this using the xElement because the xsd/cs schema doesn't support it (at the moment)
foreach(var function in fortifyScan._fvdl.ProgramData.CalledWithNoDef.xElement().elements())
{
fortifyScan.CalledWithNoDefs.Add(
new Fortify_CalledWithNoDef()
{
Name = function.attribute("name").value(),
Namespace = function.attribute("namespace").value(),
EnclosingClass = function.attribute("enclosingClass").value()
});
}
return fortifyScan;
}
public static Fortify_Scan mapSinks(this Fortify_Scan fortifyScan)
{
foreach(var sink in fortifyScan._fvdl.ProgramData.Sinks.SinkInstance)
{
fortifyScan.Sinks.Add(
new Fortify_Sink()
{
RuleID = sink.ruleID.str() ,
Function_Call = new Fortify_Function(sink.FunctionCall.Function.name,sink.FunctionCall.SourceLocation)
});
}
return fortifyScan;
}
public static Fortify_Scan mapSources(this Fortify_Scan fortifyScan)
{
foreach(var source in fortifyScan._fvdl.ProgramData.Sources.SourceInstance)
{
var fortifySource = new Fortify_Source()
{
RuleID = source.ruleID.str() ,
};
if (source.FunctionCall.notNull())
fortifySource.Function_Call = new Fortify_Function(source.FunctionCall.Function.name,source.FunctionCall.SourceLocation);
if (source.FunctionCall.notNull())
fortifySource.Function_Entry = new Fortify_Function(source.FunctionCall.Function.name,source.FunctionCall.SourceLocation);
if (source.TaintFlags.notNull())
fortifySource.TaintFlags = (from taintFlag in source.TaintFlags.TaintFlag
select taintFlag.name).toList();
fortifyScan.Sources.Add(fortifySource);
}
return fortifyScan;
}
public static Fortify_Scan mapSnippets(this Fortify_Scan fortifyScan)
{
foreach(var snippet in fortifyScan._fvdl.Snippets.Snippet)
{
fortifyScan.Snippets.Add(
new Fortify_Snippet()
{
Id = snippet.id.str() ,
Text = snippet.Text,
CodeLocation = new Fortify_CodeLocation()
{
Path = snippet.File,
Line = snippet.StartLine,
LineEnd = snippet.EndLine
}
});
}
return fortifyScan;
}
public static Fortify_Scan mapVulnerabilities(this Fortify_Scan fortifyScan)
{
foreach(var vulnerability in fortifyScan._fvdl.Vulnerabilities.Vulnerability)
if (vulnerability.notNull())
{
var fortifyVulnerability = new Fortify_Vulnerability();
//from ClassInfo
fortifyVulnerability.AnalyzerName = vulnerability.ClassInfo.AnalyzerName;
fortifyVulnerability.ClassId = vulnerability.ClassInfo.ClassID;
fortifyVulnerability.DefaultSeverity = vulnerability.ClassInfo.DefaultSeverity;
fortifyVulnerability.Kingdom = vulnerability.ClassInfo.Kingdom;
fortifyVulnerability.Type = vulnerability.ClassInfo.Type;
fortifyVulnerability.SubType = vulnerability.ClassInfo.Subtype;
//from
fortifyVulnerability.InstanceId = vulnerability.InstanceInfo.InstanceID;
fortifyVulnerability.InstanceSeverity = vulnerability.InstanceInfo.InstanceSeverity;
fortifyVulnerability.Confidence = vulnerability.InstanceInfo.Confidence;
//
//from AnalysisInfo
var analysisInfo = vulnerability.AnalysisInfo;
if (analysisInfo.Unified.notNull())
{
if (analysisInfo.Unified.Context.notNull() && analysisInfo.Unified.Context.Function.notNull())
fortifyVulnerability.Context = new Fortify_Function(analysisInfo.Unified.Context.Function.name,
analysisInfo.Unified.Context.FunctionDeclarationSourceLocation);
if (analysisInfo.Unified.ReplacementDefinitions.notNull())
foreach(var def in analysisInfo.Unified.ReplacementDefinitions.Def)
fortifyVulnerability.ReplacementDefinitions.Definitions.add(def.key, def.value);
foreach(var trace in analysisInfo.Unified.Trace)
foreach(var entry in trace.Primary.Entry)
{
var traceEntry = new Fortify_TraceEntry();
if (entry.NodeRef.notNull())
traceEntry.NodeRefId = entry.NodeRef.id;
if (entry.Node.notNull())
{
var node = entry.Node;
traceEntry.DetailsOnly = node.detailsOnly ?? false;
traceEntry.IsDefault = node.isDefault ?? false;
traceEntry.Label = node.label ?? "";
if (node.Action.notNull())
{
traceEntry.ActionType = node.Action.type;
traceEntry.ActionValue = node.Action.TypedValue;
}
if (node.Knowledge.notNull())
{
foreach(var fact in node.Knowledge.Fact)
traceEntry.KnowledgeFacts.Add(new Fortify_TraceEntryFact()
{
Primary = fact.primary,
Type = fact.type,
Value = fact.TypedValue
});
}
if (node.Reason.notNull())
{
traceEntry.Reason_RuleId = node.Reason.Rule.notNull()
? node.Reason.Rule.ruleID
: "";
traceEntry.Reason_TraceRef = node.Reason.TraceRef.notNull()
? node.Reason.TraceRef.str()
: "";
traceEntry.Reason_Internal = node.Reason.Internal.notNull()
? node.Reason.Internal.str()
: "";
}
if (node.SourceLocation.notNull())
{
traceEntry.SourceLocation = new Fortify_CodeLocation(node.SourceLocation);
traceEntry.SourceLocation_ContextId = node.SourceLocation.contextId ?? 0;
traceEntry.SourceLocation_Snippet = "node.SourceLocation.snippet";
}
if (node.SecondaryLocation.notNull())
{
traceEntry.SecundaryLocation = new Fortify_CodeLocation(node.SecondaryLocation);
traceEntry.SecundaryLocation_Snippet = node.SecondaryLocation.snippet;
}
}
fortifyVulnerability.Traces.Add(traceEntry);
}
}
fortifyScan.Vulnerabilities.add(fortifyVulnerability);
}
return fortifyScan;
}
}
Since all the data conversion is happening at the backend classes, the GUI script is now just focused on showing the data (with most objects shown on separate tabs)
//var topPanel = panel.clear().add_Panel();
var topPanel = "Util - FVDL viewer (drop *.fvdl on the left-hand-side PropertyGrid)".popupWindow(1000,500);
topPanel.insert_LogViewer();
</pre>
var tabControl = topPanel.add_TabControl();
var scannedFiles_Table = tabControl.add_Tab("Scanned Files").add_TableList();
var context_Table = tabControl.add_Tab("Contexts").add_TableList();
var description_Table = tabControl.add_Tab("Descriptions").add_TableList();
var calledWithNoDefs_Table = tabControl.add_Tab("CalledWithNoDefs").add_TableList();
var sinks_Table = tabControl.add_Tab("Sinks").add_TableList();
var sources_Table = tabControl.add_Tab("Sources").add_TableList();
var snippets_Table = tabControl.add_Tab("Snippets").add_TableList();
var vulnerabilities_Table = tabControl.add_Tab("Vulnerabilities").add_TableList();
//var traceNodes_Table = tabControl.add_Tab("TraceNodes").add_TableList();
var propertyGrid = topPanel.insert_Left().add_PropertyGrid();
Fortify_Scan fortifyScan = null;
var apiFortify = new API_Fortify();
Action<string> showUserMessage =
(message)=> {
// userMessages_Label.set_Text(message);
message.info();
};
Action<Fortify_Scan> showFvdl =
(scan) =>
{
vulnerabilities_Table.title("Showing {0} Vulnerabilties".format(scan.Vulnerabilities.size()))
.show(scan.Vulnerabilities);
scannedFiles_Table.show(scan.ScannedFiles);
context_Table.show(scan.Contexts); //.makeColumnWidthMatchCellWidth();
description_Table.show(scan.Descriptions);
calledWithNoDefs_Table.show(scan.CalledWithNoDefs);
sinks_Table.show(scan.Sinks);
sources_Table.show(scan.Sources);
vulnerabilities_Table.show(scan.Vulnerabilities);
snippets_Table.show(scan.Snippets);
// traceNodes_Table.show((from vulnerability in scan.Vulnerabilities
// select vulnerability.Traces).SelectMany(p=>p).toList());
};
Action<string> loadAndShowFile =
(file)=>{
showUserMessage("... loading file: {0}".format(file.fileName()));
O2Thread.mtaThread(()=>{
fortifyScan = apiFortify.convertToFortifyScan(file);
showFvdl(fortifyScan);
propertyGrid.show(fortifyScan);
showUserMessage("loaded fvdl file: {0}".format(file.fileName()));
});
};
propertyGrid.onDrop(loadAndShowFile);
return "done";
//O2File:API_Fortify_1_6.cs
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll
About
The O2 platform represents a new paradigm for how to perform, document and distribute Web Application security reviews. O2 is designed to Automate Security Consultants Knowledge and Workflows and to Allow non-security experts to access and consume Security Knowledge.
O2 can also be a very powerful prototyping and fast-development tool for .NET. Most O2 APIs are written using a Fluent API design, and its core has been published as a separate project called FluentSharp (hosted at CodePlex)
Download
Windows ClickOnce Installer:
Mailing List
The best place to keep updated with the lastest news and developers it to subscribe to the OWASP O2 Platform Mailing list (you can read its archives here)
OWASP O2 Platform Blog 