OWASP O2 Platform Blog

O2 Community Script: Get messages from OWASP mailman

Daniel (from the OWASP .NET Project) sent me the O2 script below which is designed to consume the data exposed by the OWASP Mailman (mailing list manager).

This is the first pass at the creation of a tool that is able to:

  • grab the data from mailman,
  • visualize it,
  • parse it, and eventually
  • export it into other mailing lists solutions (for example Google Groups)

There is still quite a lot to do here, so if you want to help, go to the https://github.com/o2platform/Scripts-by-O2-Users GitHub repository and create a fork of it.

This is what this script looks like for the Owasp-o2-platform mailing list:

This is what it looks like for the   OWASP-NYNJMetro mailing list

This is the script’s source code:

</pre>


#region config
var owaspMailmanUrl = "https://lists.owasp.org/mailman/listinfo";
var owaspArchive = "https://lists.owasp.org/pipermail/";
var owaspListInfo = "https://lists.owasp.org/mailman/listinfo/";

var o2platform = "Owasp-o2-platform";

#endregion
//var topPanel = panel.clear().add_Panel();
var topPanel = O2Gui.open<Panel>("OWASP Mailman Helper", 1200, 900);
var optionsPanel = topPanel.insert_Above<GroupBox>(100).set_Text("Options").add_Panel();
var mainPanel = topPanel.add<GroupBox>().set_Text("").add_Panel();

var threadsPanel = mainPanel.insert_Left<GroupBox>().set_Text("Threads").add_Panel();
var threadBrowser = mainPanel.insert_Left<GroupBox>().set_Text("Thread").add_Browser();

var threadTree = threadsPanel.add_TreeView();
var statsPanel = mainPanel.insert_Left<GroupBox>(400).set_Text("Statistics").add_Panel();//
var ggroupsPanel = optionsPanel.insert_Right<GroupBox>(200).set_Text("Port to google group").add_Panel();
var statPanel = optionsPanel.insert_Right<GroupBox>(500).set_Text("Mailman Group statistics").add_Panel();
Action<string> selectStastType = (list) => {list.info();};

var statsControls = statPanel.add_LabelAndComboBoxAndButton("Select statistics type: ", "", "Draw", selectStastType);
var drawStatsBtn = statsControls.controls<Button>();
var drawStatsCombo = statsControls.controls<ComboBox>();
drawStatsCombo.add_Item("Posts per year");

var statusStrip = topPanel.parentForm().add_StatusStrip(false);
//options panel
var currentList_Label = optionsPanel.insert_Below<Label>().set_Text("Selected mailing list: ...");
Action<string> selectList = (list) => {currentList_Label.set_Text("Selected mailing list: "+list);};
var tab = optionsPanel.add_LabelAndComboBoxAndButton("Select list: ", "", "Load", selectList);
var list_CombBox = tab.controls<ComboBox>();
var loadThreadsBtn = tab.controls<Button>();
var analyseAllBtn = optionsPanel.add_Button(20,20,"Analyse all lists");

var listLinks = new List<HtmlAgilityPack.HtmlNode>();

//chart control
var chart = new Chart();
chart.Dock = DockStyle.Fill;

statsPanel.add(chart);
//threads
//TODO: change from hashtable to proper class
var threads = new List<System.Collections.Hashtable>();

//reading owasp mailing lists
O2Thread.mtaThread(
    ()=>{
        lock(listLinks) {
        statusStrip.set_Text(" Loading OWASP mailman lists from OWASP...");
        var htmlDocument = owaspMailmanUrl.uri().getHtml().htmlDocument();
         
         listLinks =  htmlDocument.select("//table")[0]
                                        .SelectNodes(".//a")
                                        .toList<HtmlAgilityPack.HtmlNode>()
                                        .Where(x=>x.innerHtml().Contains("<strong>")).ToList();
        int i = 0;
        foreach(var link in listLinks) {
            var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
            var item = list_CombBox.add_Item(name);   
            if(name.eq(o2platform))
                list_CombBox.select_Item(i);
            i++;
           
        }
    statusStrip.set_Text(" OWASP mailman lists loaded!");
    }
});   
loadThreadsBtn.onClick(
    () => {
    lock(listLinks) {
        var selectedList = list_CombBox.get_Text();
        foreach(var link in listLinks)
        {
            var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
            if(name == selectedList) {
                var nLink = link.attribute("href").value().replace(owaspListInfo, owaspArchive);
                O2Thread.mtaThread(
                ()=>{
                    threads = new List<System.Collections.Hashtable>();
                    statusStrip.set_Text(" Downloading threads from "+nLink);
                    var htmlDocument = nLink.uri().getHtml().htmlDocument();
                    var threadedLinks =  htmlDocument.select("//table")[0]
                                        .SelectNodes(".//a")
                                        .toList<HtmlAgilityPack.HtmlNode>()
                                        .Where(x=>x.Attributes["href"].Value.Contains("thread.html")).ToList();
                    threadTree.clear();
                    foreach(var threadedLink in threadedLinks) {
                        var nnLink = (nLink+"/"+threadedLink.Attributes["href"].Value);
                        var html = nnLink.uri().getHtml().htmlDocument();
                        var threadLinks = html.select("//body")[0].SelectNodes(".//a").toList<HtmlAgilityPack.HtmlNode>().Where(x=>x.Attributes["href"].notNull());
                        foreach(var threadLink in threadLinks) {
                            if(threadLink.Attributes["href"].Value.regEx("[0-9]{6}\\.html")) {
                                var threadHtml = nnLink.replace("thread.html",threadLink.Attributes["href"].Value).url().getHtml().htmlDocument();
                                var h = new System.Collections.Hashtable();
                                h["Topic"] =  threadHtml.select("//h1")[0].InnerHtml;
                                h["Author"] =  threadHtml.select("//b")[0].InnerHtml;
                                var timeStr =  threadHtml.select("//i")[0].InnerHtml;
                                h["Content"] = threadHtml.select("//pre")[0].InnerHtml;
                                h["Time"] = DateTime.ParseExact(timeStr.Replace(" EST", "-5:00").Replace(" EDT","-4:00").Replace("  ", " "),"ddd MMM d HH:mm:sszzz yyyy", System.Globalization.CultureInfo.InvariantCulture);
                                threads.Add(h);
                                var node = threadTree.add_Node(h["Topic"]);
                                node.add_Node("Time: {0}".format(h["Time"]));
                                node.add_Node("Author: {0}".format(h["Author"]));
                                node.set_Tag(h["Content"]);
                            }
                        }
                    }
                    statusStrip.set_Text("Threads downloaded from "+nLink);
                });
            }
        }
    }
});
analyseAllBtn.onClick(
    () => {
        threadTree.clear();
        foreach(var link in listLinks) {
            var name = link.innerHtml().replace("<strong>", "").replace("</strong>","");
            var nLink = link.attribute("href").value().replace(owaspListInfo, owaspArchive);
            var node = threadTree.add_Node(name);
            node.add_Node("Link: {0}".format(nLink));
            var htmlDocument = nLink.uri().getHtml().htmlDocument();
            var trs =  htmlDocument.select("//table")[0]
                                        .SelectNodes(".//tr")
                                        .toList<HtmlAgilityPack.HtmlNode>().ToList();
            var first = "";
            var last = "";
            foreach(var tr in trs) {
                var td = tr.SelectNodes(".//td")[0];
                if(first=="")
                    first = td.InnerText;
                last = td.InnerText;
                td.InnerText.log();
            }
            node.add_Node("First post: {0}".format(first));
            node.add_Node("Last post: {0}".format(last));
           
        }
    }
);
threadTree.afterSelect(
    (treeNode)=> {
        if(treeNode.get_Tag().notNull())
            threadBrowser.set_Text("<h2>{1}</h2><h3>{2}</h3> <h3>{3}</h3><hr /><br />{0}".format(treeNode.get_Tag().str(), treeNode.get_Text(), treeNode.Nodes[0].get_Text(), treeNode.Nodes[1].get_Text()));
});
drawStatsBtn.onClick(
    () => {
        statusStrip.set_Text(" Drawing new chart...");
        var dict = new Dictionary<int, int>();
        if(threads.count()==0) {
            statusStrip.set_Text(" No threads for statistics...");
            return;
        }
        foreach(var de in threads) {
            var time = (System.DateTime)de["Time"];
            if(!dict.Keys.Contains(time.Year))
                dict[time.Year] = 0;
            dict[time.Year]++;
        }
        chart.Series.Clear();
        foreach(var k in dict.Keys) {
            var serie = new Series(k.str());
            serie.ChartType = SeriesChartType.Bar;
            serie.Points.AddXY(k,dict[k]);
            serie.IsValueShownAsLabel = true;
            chart.Series.Add(serie);
        }
        chart.ChartAreas.Add("Chart");
        statusStrip.set_Text(" Drawing finished...");
    }
);
return "ok";

//using O2.XRules.Database.Utils.ExtensionMethods
//using O2.External.IE.ExtensionMethods
//using System.Windows.Forms.DataVisualization.Charting

//O2File:HtmlAgilityPack_ExtensionMethods.cs
//O2File:IE_JQuery.cs

//O2Ref:WatiN.Core.1x.dll
//O2Ref:O2_External_IE.dll
//O2Ref:O2_Misc_Microsoft_MPL_Libs.dll
//O2Ref:System.Xml.dll
//O2Ref:System.Xml.Linq.dll
//O2Ref:System.Windows.Forms.DataVisualization.dll


 

August 2, 2011 Posted by | .NET, Interoperability | Leave a comment