I often need to look at logs. I needed a way to quickly highlight different parts of the logs. I created a plugin for Sublime Text to do that.
Here are instructions to create a plugins.
Here is my plugin:
You can invoke it with
view.run_command("reggae", {"pattern_args": ["pattern 1", "pattern 2"]})
limit_req_zone $binary_remote_addr
cut -d "\"" -f1,8 fdpapi_ssl_access_w2_21.log | cut -d " " -f4,6 | sed 's/\[//' | sed 's/,//' | sed 's/\"//' > fdpapi_ssl_access_w2_21.csv 
library(readr) head(logrows)
| X1 | X2 |
|---|---|
| 2017-02-19 00:00:04 | 100.0.XX.244 |
| 2017-02-19 00:00:04 | 100.0.XX.244 |
| 2017-02-19 00:03:51 | 100.1.XXX.223 |
| 2017-02-19 00:03:51 | 100.1.XXX.223 |
| 2017-02-19 00:03:52 | 100.1.XXX.223 |
| 2017-02-19 00:02:48 | 100.1.XXX.60 |
tb <- table(logrows$X2) # x is a vector with the datetimes for a certain IP # Intermediary steps findmaxfreq <- function(x) { ipsagg <- aggregate( logrows$X1 ~ ipsfactor, FUN = findmaxfreq ) head(ipsagg[order(ipsagg$`logrows$X1`, decreasing = TRUE),]) | ipsfactor | logrows$X1 |
|---|---|
| 51.XXX.56.29 | 1160 |
| 64.XX.231.130 | 711 |
| 64.XX.231.169 | 705 |
| 68.XX.69.191 | 500 |
| 217.XXX.203.205 | 462 |
| 64.XX.231.157 | 458 |
quantile(ipsagg$`logrows$X1`, seq(from=0.99, to=1, by=0.001)) 
ipsaggrpm <- ipsagg$`logrows$X1` 
limit_req_zone $binary_remote_addr zone=one:10m rate=63r/m; limit_req_zone $binary_remote_addr zone=one:10m rate=32r/m;
Set-StrictMode -version 1
Function CreateNode($id, $label, $xmlDoc, $nodes, $rootNS, $ZeroAttributeValue, $OneAttributeValue, $comment)
{
$node = $xmlDoc.CreateElement("node", $rootNS)
$node.SetAttribute("id",$id)
$node.SetAttribute("label",$label)
if ($ZeroAttributeValue) { # occurrences
$attvalues = $xmlDoc.CreateElement("attvalues", $rootNS)
$attvalue = $xmlDoc.CreateElement("attvalue", $rootNS)
$attvalue.SetAttribute("for","0")
$attvalue.SetAttribute("value",$ZeroAttributeValue)
$attvalue.AppendChild($xmlDoc.CreateComment('occurrences'))
$attvalues.AppendChild($attvalue)
$node.AppendChild($attvalues)
}
if ($OneAttributeValue) { # URL
$attvalues = $xmlDoc.CreateElement("attvalues", $rootNS)
$attvalue = $xmlDoc.CreateElement("attvalue", $rootNS)
$attvalue.SetAttribute("for","1")
$attvalue.SetAttribute("value",$OneAttributeValue)
$attvalue.AppendChild($xmlDoc.CreateComment('URL'))
$attvalues.AppendChild($attvalue)
$node.AppendChild($attvalues)
}
$node.AppendChild($attvalues)
$node.AppendChild($xmlDoc.CreateComment($comment))
$nodes.AppendChild($node)
}
Function CreateEdge($id, $source, $target, $xmlDoc, $edges, $rootNS)
{
$edge = $xmlDoc.CreateElement("edge", $rootNS)
$edge.SetAttribute("id",$edgeid)
$edge.SetAttribute("source",$source)
$edge.SetAttribute("target",$target)
$edges.AppendChild($edge)
}
Function CreateAttributes($xmlDoc, $graph, $rootNS)
{
$attribs = $xmlDoc.CreateElement("attributes", $rootNS)
$attribs.SetAttribute("class","node")
$attrib = $xmlDoc.CreateElement("attribute", $rootNS)
$attrib.SetAttribute("id","0")
$attrib.SetAttribute("title","occurrences-node")
$attrib.SetAttribute("type","integer")
$attribs.AppendChild($attrib)
$attrib = $xmlDoc.CreateElement("attribute", $rootNS)
$attrib.SetAttribute("id","1")
$attrib.SetAttribute("title","url")
$attrib.SetAttribute("type","string")
$attribs.AppendChild($attrib)
$graph.AppendChild($attribs)
}
Function EqualString($str1, $str2) {
return !$str1.CompareTo($str2)
}
Function WhiteListTag($tag) {
$lang = !'português'.CompareTo($tag) -or !'español'.CompareTo($tag)
$black = (EqualString 'CSN_' $tag) -or (EqualString 'CSN_source' $tag) -or (EqualString 'CSN_f' $tag) -or (EqualString 'CSN_t_' $tag) -or (EqualString 'CSN_freepage' $tag)
$hasPrefix = $tag.StartsWith('CSN_')
return $lang -or ( !$black -and $hasPrefix )
}
[void][system.reflection.assembly]::LoadFrom("E:\install\JsonNetBin\Net35\Newtonsoft.Json.dll")
Add-Type -AssemblyName "System.Net"
$json = ""
Get-Content C:\Users\gustavo.frederico\Documents\jsonDeliciousGF.txt |foreach{$json += $_ + "`r`n"}
$rss = [Newtonsoft.Json.Linq.JObject]::Parse($json)
[xml]$xmlDoc = New-Object system.Xml.XmlDocument
$xmlDoc.LoadXml('')
[System.Xml.XmlNamespaceManager] $nsmgr = $xmlDoc.NameTable
$rootNS = $xmlDoc.DocumentElement.NamespaceURI
$meta = $xmlDoc.CreateElement("meta", $rootNS)
$meta.SetAttribute("lastmodifieddate","2012-09-20")
$xmlDoc.LastChild.AppendChild($meta)
$creator = $xmlDoc.CreateElement("creator", $rootNS)
$creator.AppendChild($xmlDoc.CreateTextNode('CSNombre script'));
$meta.AppendChild($creator)
$desc = $xmlDoc.CreateElement("description", $rootNS)
$desc.AppendChild($xmlDoc.CreateTextNode('gexf representation of Delicious entries'));
$meta.AppendChild($desc)
$graph = $xmlDoc.CreateElement("graph", $rootNS)
$graph.SetAttribute("defaultedgetype","directed")
$graph.SetAttribute("mode","static")
CreateAttributes $xmlDoc $graph $rootNS
$nodes = $xmlDoc.CreateElement("nodes", $rootNS)
$xmlDoc.LastChild.AppendChild($graph)
$csnProps = $rss.Properties() | where { WhiteListTag $_.Name.ToString() }
$existingLabelNodesTags = @{}
# key: tag name , value: tag node id
$existingLabelNodesURLs = @{}
# key: URL , value: url node id.
$tagNodeId = 0
$edgeid = 0
foreach($prop in $csnProps) {
$existingLabelNodesTags.Add($prop.Name.ToString(), $tagNodeId)
CreateNode $tagNodeId $prop.Name.ToString() $xmlDoc $nodes $rootNS $prop.Value.ToString() $null 'tag/label node'
$tagNodeId++
# Write-Host $prop.Value.ToString()
}
$wc = New-Object System.Net.WebClient
$edges = $xmlDoc.CreateElement("edges", $rootNS)
$URLNodeId = $tagNodeId
foreach($prop2 in $csnProps) {
$currentTag = $prop2.Name.ToString()
$tagNodeIdForCurrentTag = $existingLabelNodesTags.Get_Item($currentTag)
$rawJson = $wc.DownloadString("http://feeds.delicious.com/v2/json/gcsfred/" + $currentTag) # "?count=99"
$rawJson = "{ array: " + $rawJson + " }"
$references = [Newtonsoft.Json.Linq.JObject]::Parse($rawJson)
$referencesProps = $references.Properties()
foreach($refProp in $references['array']) {
$uVal = $refProp['u'].ToString()
$dVal = $refProp['d'].ToString()
# create node for u , d if it doesn't exist
if ( $existingLabelNodesURLs.ContainsValue($uVal) ) {
$nodeIdURL = $existingLabelNodesURLs.Get_Item($uVal)
} else {
$existingLabelNodesURLs.Set_Item($uVal, $URLNodeId)
$nodeIdURL = $URLNodeId
# create node for u and d
CreateNode $nodeIdURL $dVal $xmlDoc $nodes $rootNS $null $uVal 'URL node'
$URLNodeId++
}
# now, create edge between currentTag node and URL nodes
CreateEdge $edgeid $tagNodeIdForCurrentTag $nodeIdURL $xmlDoc $edges $rootNS
$edgeid++
# end of u and d
$otherTags = $refProp['t']
$i=0
while( $i -lt $otherTags.Count) {
$otherTag = $otherTags[$i].ToString()
$isWhite = WhiteListTag $otherTag
if (!$isWhite) {
$i++
continue
}
# fetch node id of $otherTag
$nodeIdOtherTag = $existingLabelNodesTags.Get_Item($otherTag)
CreateEdge $edgeid $nodeIdURL $nodeIdOtherTag $xmlDoc $edges $rootNS
$edgeid++
$i++
}
}
# end for each reference tag
}
$graph.AppendChild($nodes)
$graph.AppendChild($edges)
$xmlDoc.Save("c:\mytemp\foo.xml")
#include//winsock
#include//DNS api's
#include//standard i/o
#include
//Usage of the program
void Usage(char *progname) {
fprintf(stderr,"Usage\n%s -n [HostName|IP Address] -t [Type] -s [DnsServerIp]\n",progname);
fprintf(stderr,"Where:\n\t\"HostName|IP Address\" is the name or IP address of the computer ");
fprintf(stderr,"of the record set being queried\n");
fprintf(stderr,"\t\"Type\" is the type of record set to be queried A or PTR\n");
fprintf(stderr,"\t\"DnsServerIp\"is the IP address of DNS server (in dotted decimal notation) ");
fprintf(stderr,"to which the query should be sent\n");
exit(1);
}
char* WChar2char(wchar_t * src, char * ppDest) {
size_t convertedChars = 0;
size_t origsize = wcslen(src) + 1;
wcstombs_s(&convertedChars, ppDest, origsize, src, _TRUNCATE);
return (char*)ppDest;
}
void Char2Wchar(char* src, wchar_t * ppDest) {
size_t convertedChars = 0;
size_t origsize = strlen(src) + 1;
mbstowcs_s(&convertedChars, ppDest, origsize, src, _TRUNCATE);
}
void ReverseIP(wchar_t* pIP)
{
wchar_t seps[1]; seps[0] = (wchar_t)'.';
wchar_t *token;
char pIPSec[4][4];
int i=0;
token = wcstok( pIP, seps);
while( token != NULL )
{
/* While there are "." characters in "string" */
sprintf(pIPSec[i],"%s", token);
/* Get next "." character: */
token = wcstok( NULL, seps );
i++;
}
swprintf(pIP,L"%s.%s.%s.%s.%s", pIPSec[3],pIPSec[2],pIPSec[1],pIPSec[0],"IN-ADDR.ARPA");
}
// the main function
void __cdecl main(int argc, char *argv[])
{
DNS_STATUS status; //Return value of DnsQuery_A() function.
PDNS_RECORD pDnsRecord; //Pointer to DNS_RECORD structure.
PIP4_ARRAY pSrvList = NULL; //Pointer to IP4_ARRAY structure.
WORD wType; //Type of the record to be queried.
wchar_t pOwnerName[255]; //Owner name to be queried.
wchar_t pReversedIP[255]; //Reversed IP address.
wchar_t DnsServIp[255]; //DNS server ip address.
DNS_FREE_TYPE freetype ;
freetype = DnsFreeRecordListDeep;
IN_ADDR ipaddr;
char * chOwnerName = NULL;
if(argc > 4) {
for(int i = 1; i < argc ; i++) {
if ( (argv[i][0] == '-') || (argv[i][0] == '/') ) {
switch(tolower(argv[i][1])) {
case 'n':
//pOwnername=
chOwnerName = argv[++i];
Char2Wchar(chOwnerName, (wchar_t *) pOwnerName);
break;
case 't':
if (!stricmp(argv[i+1], "A") )
wType = DNS_TYPE_A; //Query host records to resolve a name.
else if (!stricmp(argv[i+1], "PTR") )
{
//pOwnerName should be in "xxx.xxx.xxx.xxx" format
if(wcslen(pOwnerName)<=15)
{
//You must reverse the IP address to request a Reverse Lookup
//of a host name.
swprintf(pReversedIP,L"%s",pOwnerName);
ReverseIP(pReversedIP);
size_t sizeRev = wcslen(pReversedIP) + 1;
wcsncpy_s(pOwnerName, sizeRev, pOwnerName, 255);
//pOwnerName=pReversedIP;
wType = DNS_TYPE_PTR; //Query PTR records to resolve an IP address
}
else
{
Usage(argv[0]);
}
}
else
Usage(argv[0]);
i++;
break;
case 's':
// Allocate memory for IP4_ARRAY structure.
pSrvList = (PIP4_ARRAY) LocalAlloc(LPTR,sizeof(IP4_ARRAY));
if(!pSrvList){
printf("Memory allocation failed \n");
exit(1);
}
if(argv[++i]) {
wchar_t aarray[300];
Char2Wchar(argv[i], (wchar_t*)aarray);
wcscpy(DnsServIp,aarray);
char tmp[500];
pSrvList->AddrCount = 1;
pSrvList->AddrArray[0] = inet_addr(WChar2char(DnsServIp, (char*) tmp)); //DNS server IP address
break;
}
default:
Usage(argv[0]);
break;
}
}
else
Usage(argv[0]);
}
}
else
Usage(argv[0]);
// Calling function DnsQuery to query Host or PTR records
status = DnsQuery(pOwnerName, //Pointer to OwnerName.
wType, //Type of the record to be queried.
DNS_QUERY_BYPASS_CACHE, // Bypasses the resolver cache on the lookup.
pSrvList, //Contains DNS server IP address.
&pDnsRecord, //Resource record that contains the response.
NULL); //Reserved for future use.
if (status){
if(wType == DNS_TYPE_A)
printf("Failed to query the host record for %s and the error is %d \n", pOwnerName, status);
else
printf("Failed to query the PTR record and the error is %d \n", status);
} else {
if(wType == DNS_TYPE_A) {
//convert the Internet network address into a string
//in Internet standard dotted format.
ipaddr.S_un.S_addr = (pDnsRecord->Data.A.IpAddress);
printf("The IP address of the host %s is %s \n", pOwnerName,inet_ntoa(ipaddr));
// Free memory allocated for DNS records.
DnsRecordListFree(pDnsRecord, freetype);
}
else {
printf("The host name is %s \n",(pDnsRecord->Data.PTR.pNameHost));
// Free memory allocated for DNS records.
DnsRecordListFree(pDnsRecord, freetype);
}
}
LocalFree(pSrvList);
}
Could not load type 'Blah.Control'. at System.Web.UI.TemplateParser.GetType(String typeName, Boolean ignoreCase, Boolean throwOnError)
at System.Web.UI.TemplateParser.ProcessInheritsAttribute(String baseTypeName, String codeFileBaseTypeName, String src, Assembly assembly)
at System.Web.UI.TemplateParser.PostProcessMainDirectiveAttributes(IDictionary parseData)
