#!/usr/bin/ruby
#
require 'csv'
 
input = "export.csv"
output = "gcal.csv"
 
csvfile = File.open(input) {|f| f.read}
 
puts "Parsing..."
 
csv = CSV::parse(csvfile)
 
fields = csv.shift
 
puts "Writing..."
File.open(output, "w") do |f|
   f.print "Subject, Start Date, Start Time, End Date, End Time\n"
   csv.each do |line|
     startdate, starttime = Time.at(line[6].to_i).strftime("%m/%d/%Y,%I:%M:%S %p").split(',')
     enddate, endtime = Time.at(line[7].to_i).strftime("%m/%d/%Y,%I:%M:%S %p").split(',')
     f.print "\"#{line[11]}\",#{startdate},#{starttime},#{enddate},#{endtime}\n"
   end
end
 
puts "Done."

#!/usr/bin/ruby
#
#
require 'win32ole'
require 'dbi'
 
class Access
   attr_accessor :mdb, :conn, :data, :fields
 
   def initialize(mdb=nil)
       @mdb = mdb
       @conn = nil
       @data = nil
       @fields = nil
   end
 
   def open
       connstring = "Provider=Microsoft.Jet.OLEDB.4.0;Data Source=#{@mdb}"
       @conn = WIN32OLE.new('ADODB.Connection')
       @conn.Open(connstring)
   end
 
   def query(sql)
       set = WIN32OLE.new('ADODB.Recordset')
       set.Open(sql, @conn)
       @fields = []
       set.Fields.each do |field|
           @fields << field.Name
       end
       @data = set.GetRows.transpose
       set.Close
   end
 
   def close
       @conn.Close
   end
end
 
output = "gcal.csv"
 
rows = Array.new
 
db = Access.new('c:\path\to\mdb')
db.open
 
db.query("SELECT * FROM Main;")
names = db.fields
rows = db.data
 
#Alternatively
DBI.connect("DBI:ODBC:driver=Microsoft Access Driver (*.mdb);"+"dbq=c:/path/to/mdb") do |dbh|
   dbh.select_all('select * from Main') {|row| rows << row}
end
 
puts "Writing..."
File.open(output, "w") do |f|
   f.print "Subject, Start Date, Start Time, End Date, End Time\n"
   rows.each do |line|
     startdate, starttime = Time.at(line[6].to_i).strftime("%m/%d/%Y,%I:%M:%S %p").split(',')
     enddate, endtime = Time.at(line[7].to_i).strftime("%m/%d/%Y,%I:%M:%S %p").split(',')
     f.print "\"#{line[11]}\",#{startdate},#{starttime},#{enddate},#{endtime}\n"
   end
end
 
puts "Done."

access=# \d main
                 TABLE "public.main"
     COLUMN     |          Type          | Modifiers 
----------------+------------------------+-----------
 record_id      | bigint                 | NOT NULL
 STATUS         | integer                | 
 placement      | bigint                 | 
 private        | smallint               | 
 category       | character varying(20)  | 
 start_time     | bigint                 | 
 end_time       | bigint                 | 
 untimed        | smallint               | 
 time_zone      | character varying(40)  | 
 location       | character varying(255) | 
 summary        | text                   | 
 alarm_advance  | character varying(10)  | 
 alarm_unit     | character varying(10)  | 
 repeated_event | character varying(255) | 
 alarm          | smallint               | 
 note           | character varying(100) | 
access=#

access=$ SELECT count(*) FROM main;
 count 
-------
  5094
(1 row)
access=$ SELECT count(DISTINCT record_id) FROM main;
 count 
-------
  5074
(1 row)
access=$ SELECT count(DISTINCT start_time) FROM main;
 count 
-------
  2488
(1 row)
access=$ SELECT count(DISTINCT end_time) FROM main;
 count 
-------
  2490
(1 row)
access=$ SELECT count(DISTINCT summary) FROM main;
 count 
-------
  2264
(1 row)
access=$ SELECT record_id, start_time, end_time, summary 
FROM main 
WHERE record_id IN 
    (SELECT record_id 
     FROM main 
     GROUP BY record_id 
     HAVING count(*)>1);
 record_id | start_time |  end_time  |                                 summary                                 
-----------+------------+------------+-------------------------------------------------------------------------
         0 | 1231437600 | 1231441200 | tammy 
         0 | 1231869600 | 1231873200 | nb chanber lunch
         0 | 1229642100 | 1229645700 | tammy AND joe photos st claire broiler
         0 | 1231959600 | 1231963200 | dr hunt
         0 | 1230505200 | 1230508800 | tilsen photos
         0 | 1230568200 | 1230571800 | meet gary at studio
         0 | 1230571800 | 1230584400 | bri AND kids
         0 | 1230744600 | 1230748200 | tilsen, AND sandy ORDER y membership mail
         0 | 1230681600 | 1230681600 | Dan, missy AND the kids.
         0 | 1231610400 | 1231614000 |  james j hill houseOngoing Daily 11/15/08 - 2/22/09  m-sat 10-4 sun 1-4
         0 | 1230663600 | 1230667200 | tammys house glasses shopping
         0 | 1229727600 | 1229731200 | ryan help at studio
         0 | 1231889400 | 1231893000 | 
         0 | 1231889400 | 1231903800 | EMS 
         0 | 1237161600 | 1237161600 | spring break
         0 | 1229983200 | 1229986800 | msp WITH the girls
         0 | 1241049600 | 1241049600 | DISH
         0 | 1232233200 | 1232244000 | jordan senior photos excel AND studio 
         0 | 1230055200 | 1230058800 | paige studio
         0 | 1230314400 | 1230318000 | amanda tg
         0 | 1229968800 | 1229972400 | sara AND nolan 
(21 rows)
 
access=$ SELECT record_id, start_time, end_time, summary 
FROM main 
ORDER BY start_time 
ASC LIMIT 10;
 record_id | start_time | end_time | summary 
-----------+------------+----------+---------
   7128069 |   31449600 | 31449600 | c
   7128068 |   31449600 | 31449600 | a
   7123605 |   31449600 | 31449600 | a
   7128070 |   31449600 | 31449600 | 3
   7124866 |   31449600 | 31449600 | c
   7124107 |   31449600 | 31449600 | 3
   7124145 |   31449600 | 31449600 | o
   7124141 |   31449600 | 31449600 | ;
   7128072 |   31449600 | 31449600 | ;
   7128071 |   31449600 | 31449600 | o
(10 rows)
access=$ SELECT record_id, start_time, end_time, summary FROM main ORDER BY start_time DESC LIMIT 10;
 record_id | start_time |  end_time  |           summary           
-----------+------------+------------+-----------------------------
   7127485 | 1256774400 | 1256774400 | lawerance wedding
   7125815 | 1256774400 | 1256774400 | lawerance wedding
   7128114 | 1244167200 | 1244170800 | NB senior ALL night party
   7125941 | 1242489600 | 1242493200 | nyquist edding
   7125827 | 1242489600 | 1242493200 | nyquist edding
         0 | 1241049600 | 1241049600 | DISH
   7128073 | 1238079600 | 1238083200 | books IN the woods
   7125623 | 1238079600 | 1238083200 | books IN the woods
   7125697 | 1238025600 | 1238025600 | gunflint books IN the woods
   7126175 | 1238025600 | 1238025600 | gunflint books IN the woods
(10 rows)
 
access=$

access=$ SELECT DISTINCT a.start_time, a.end_time, a.summary 
INTO holdkey 
FROM main a
WHERE EXISTS 
    ( SELECT 'x' FROM main b WHERE a.start_time = b.start_time
      AND a.end_time = b.end_time
      AND a.summary = b.summary) 
ORDER BY a.start_time DESC;
SELECT
access=$ SELECT count(*) FROM holdkey;
 count 
-------
  2597
(1 row)
access=$ DELETE FROM main 
USING holdkey 
WHERE main.start_time = holdkey.start_time 
    AND main.end_time = holdkey.end_time 
    AND main.summary = holdkey.summary;
DELETE 5085
 
access=$ SELECT record_id, start_time, end_time, summary FROM main;
 record_id | start_time |  end_time  | summary 
-----------+------------+------------+---------
   5280360 |   31536000 |   31536000 | 
   5280298 |   31536000 |   31536000 | 
   5280429 |   31536000 |   31536000 | 
   7125497 | 1193437800 | 1193437800 | 
   7128378 |   31536000 |   31536000 | 
   7128376 |   31536000 |   31536000 | 
   7128374 |   31536000 |   31536000 | 
   7127620 | 1193437800 | 1193437800 | 
         0 | 1231889400 | 1231893000 | 
(9 rows)
access=$ DROP TABLE main;
DROP TABLE
access=$ SELECT * INTO main FROM holdkey;
SELECT

access=$ DELETE FROM main t1
USING main 
WHERE EXISTS 
    (SELECT * FROM main t2 
         WHERE t1.start_time = t2.start_time 
         AND t1.end_time = t2.end_time 
         AND t1.summary = t2.summary 
         AND t1.record_id < t2.record_id);
DELETE 2488
 
access=$ SELECT count(*) FROM test;
 count 
-------
  2606
(1 row)

#Rips data from Palm Desktop.  Uploads it to Google Calendar
#Written with Python 2.5 (though imports should work anyway)
#
#Currently, the Access MDB Palm Datebook uses has been exported to a 
#PostgreSQL server via ODBC, so I'll be connecting to that
#
#There's code in here for getting through Access also, but I haven't tested it.
#Use at your own risk (kinda like Access).
#
#This is mostly due to the Postgres ODBC driver, and the fact that I didn't
#want to bother with quoting all the queries for Postgres to allow spaces
 
try:
    from xml.etree import ElementTree #Python 2.5, probably 2.6/3.0 also
except ImportError:
    from elementtree import ElementTree #Python <2.4
import gdata.calendar.service
import gdata.service
import atom.service
import gdata.calendar
import atom
import getopt
import sys
import string
import time
import psycopg2 #Talk to Postgres
 
class Struct:
    def __init__(self, *args, **kwargs):
        for k,v in kwargs.items():
            setattr(self, k, v)
 
class GCalMigrate:
    def __init__(self):
        self.conn = None
        self.cur = None
        self.calendar = None
        self.records = []
 
    def connect(self):
       try:
           self.conn = psycopg2.connect("dbname='whatever' user='yournamehere' host='server'")
       except:
           print "Can't connect to the database!\n"
           sys.exit()
       self.cur = conn.cursor()
       query()
 
    def accessconnect(self,mdbpath):
        import odbc
        self.conn = odbc.odbc("driver=Microsoft Access Driver (*.mdb);DBQ=%s") % mdbpath
        self.cur = conn.cursor()
        queryaccess()
 
    def queryaccess(self):
        rows = []
        self.cur.execute("SELECT Main.[Start Time], Main.[End Time], Main.[Summary] FROM Main")
        rows = cur.fetchall()
        conn.close()
        parserows(rows)
 
    def query(self):
       rows = []
       try:
           self.cur.execute("SELECT start_time, end_time, summary FROM main")
           rows = cur.fetchall()
       except:
           print "Couldn't query the database.\n"
       conn.close()
       parserows(rows)
 
    def parserows(self, rows):
        for row in rows:
            starttime = time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime(row[0]))
            endtime = time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime(row[1]))
            title = row[2]
            record = Struct(start_time=starttime, end_time=enddtime, title=title)
            self.records.append(record)
        login()
 
    def login(self, username, password):
        self.calendar = gdata.calendar.service.CalendarService()
        self.calendar.email = username
        self.calendar.password = password
        self.calendar.source = "Palm_Desktop_Migrator"
        self.calendar.ProgrammaticLogin()
        batchsubmit()
 
 
    def batchsubmit(self):
        feed = gdata.calendar.CalendarEventFeed()
 
        for record in records:
            insertme = gdata.calendar.CalendarEventEntry()
            insertme.title = atom.Title(record.title)
            insertme.content = atom.Content("")
            insertme.when.append(gdata.calendar.When(start_time=record.start_time, end_time=record.end_time))
            insertme.batch_id = gdata.BatchId(text='Palm_Migration')
 
            feed.Add_Insert(entry=insertme)
        response = self.calendar.ExecuteBatch(feed, gdata.calendar.service.DEFAULT_BATCH_URL)
        return response
 
if __name__ == "__main__":
    runner = GCalMigrate()
    responses = runner.connect()
    for entry in responses.entry:
        print "Batch ID: %s" % entry.batch_id.text
        print "Status: %s" % entry.batch_status.code
        print "Reason: %s" % entry.batch_status.reason

#!/usr/bin/ruby
 
require 'open-uri'
require 'time'
 
class Comment
  attr_accessor :author, :datewritten, :comment
  def initialize (author, datewritten, comment)
      @author, @datewritten, @comment = author, datewritten, comment
  end
end
 
class Post
  attr_accessor :author, :title, :datewritten, :body, :comments
  def initialize (author, title, datewritten, body)
    @author, @title, @datewritten, @body = author, title, datewritten, body
    @comments = []
  end
  def addcomment (author, datewritten, comment)
    @comments.push(Comment.new(author, datewritten, comment))
  end
end
 
class Ripper
  def initialize  
    @pages = []
    @posts = Array.new
  end
  def get (uri)
    connection = open(uri)  
    content = connection.read
    return content
  end
 
  def parse (uri)
    content = get(uri)
    #blogContentInfo points to links to posts
    links = content.scan(/class="blogContentInfo">.*?<a href=".*?">/m)
    links.each do |link|
        #Strip out the bullshit amazon links
        unless link =~ /amazon/
            #Pull the URL out of the link
            link = (/.*<a href="(.*)">/m).match(link)[1]
            @pages.push(link)
        end
    end
    #Checking if there are any older pages with a hyperlink
    if content =~ /\[.*?<a href="(.*?)">Older<\/a>/
      #If so, call itself recursively to pull out the rest
      #Myspace breaks the URI standard.  Replace the spaces with real escapes
      parse($1.gsub(/\s/, "%20"))
    else
      #Edge case to break out of the loop for when there aren't any more older
      parsepages()
    end
  end
 
  def parsepages()
    @pages.each do |uri|
      #Replace with yourself, if you want
      author = "Ryan"
      content = get(uri)
      #Pull out the fields I want
      title = (/blogSubject">(.*?)\n/m).match(content)[1]
      body = (/blogContent">(.*?)<table/m).match(content)[1]
      datewritten = (/blogTimeStamp">(.*?)<\/p>/m).match(content)[1].gsub(/(^\s+|\n+)/, "")
      time = (/blogContentInfo"><b>.*?(\d+:\d+)/m).match(content)[1]
      datewritten = datewritten + " #{time}:00"
      #Parse the time, and force it into something Wordpress can deal with
      t = Time.parse(datewritten)
      datewritten = t.strftime("%m/%d/%Y %H:%M:%S")
      puts "Title: #{title}\n"
      #Create a new Post object
      post = Post.new(author, title, datewritten, body)
      #Pull out an array of all the comment blocks
      comments = content.scan(/id="blogComments.*?commentSpacer/m)
      #Pass off the post object along with the list of comments
      parsecomments(comments, post)
    end
  end
 
  def parsecomments(comments, post)
    comments.each do |com|
      author = (/profileLinks">(.*?)</m).match(com)[1]
      puts "Author: #{author}\n"
      #MySpace decided to make the CSS ids identical here, except that the
      #actual comment doesn't have "Posted" after the closing tag
      #Filter it as such
      comment = (/blogCommentsContent">(.*?)<\/p>/m).match(com)[1]
      datewritten = (/blogCommentsContent">Posted by.*?> on(.*?)<b/m).match(com)[1].gsub(/\n|\t|\r/, "")
      t = Time.parse(datewritten)
      #The same datetime munging as before
      datewritten = t.strftime("%m/%d/%Y %H:%M:%S")
      #Commit each commment to our post object
      post.addcomment(author, datewritten, comment)
    end
  #Push them all into our class array
  @posts.push(post)
  end
 
  def print(file)
    @posts.each do |post|
      #Using Movable Type's export syntax, so I don't need to mess with XML
      #It's documented here: http://www.sixapart.com/moveabletype/docs/mtimport#example
      #Basically, 5 hyphens separates the categories
      #Eight hyphens separate each post
      file.puts "TITLE: #{post.title}"
      file.puts "AUTHOR: #{post.author}"
      file.puts "DATE: #{post.datewritten}"
      #Change this, too, if you want
      file.puts "CATEGORY: MySpace"
      file.puts "-----"
      #Get rid of empty lines and fucking Windows ^M newlines, plus convert &nbsp; to " "
      file.puts "BODY:\n#{post.body.gsub(/^(\s+|\t+|\n+)$/, "").gsub(/\015/, "").gsub(/&nbsp;/, " ")}"
      file.puts "-----"
      post.comments.each do |com|
        #More stuff is possible here, but isn't necessary
        file.puts "COMMENT:"
        file.puts "AUTHOR: #{com.author}"
        file.puts "DATE: #{com.datewritten}"
        file.puts "#{com.comment}"
        file.puts "-----"
      end
      file.puts "--------"
    end
  end
end
 
#Instantiate it
ripper = Ripper.new
#Parse my blog (substitute whatever yours is here)
ripper.parse("http://blog.myspace.com/lykurgos")
#Output it
 
output = File.open("posts.txt", "a")
ripper.print(output)
puts "Done!\n"
#Import into Wordpress!

string regexPattern = @".*?\)\s
                      (?<system>\S+?)
                      :\s
                      (?<tape>\w)
                      .*,\s
                      (?<initials>.*)";
Regex re = new Regex(regexPattern, RegexOptions.ExplicitCapture);

	<form action="http://www.climate.weatheroffice.ec.gc.ca/climateData/Interform.cfm" method="post" name="stnRequest1">
		<input type="Hidden" name="hlyRange" value="N/A">
		<input type="Hidden" name="dlyRange" value="1998-4-1|2007-11-30">
		<input type="Hidden" name="mlyRange" value="1998-4-1|2007-11-1">
		<input type="Hidden" name="StationID" value="10700">
		<input type="Hidden" name="prov" value="CA">
		<input type="Hidden" name="urlExtension" value="_e.html">
	<tr id="dataTableOddRow">
		<td id="dataTableRowHeader">(AE) BOW SUMMIT</td>
		<td id="dataTableRowHeader"><abbr title="ALBERTA">ALTA</abbr></td>
		<td>
			<select name="timeframe" size="1" class="formElement75w" onChange="elementChange(document.stnRequest1,1)">
	<option value="2">Daily</option><option value="3">Monthly</option><option value="4">Almanac</option>
			</select>
		</td>
	<td>
	<select name="day" size="1" class="formElement" disabled><option value="1" >1</option><option value="2" >2</option><option value="3" >3</option><option value="4" >4</option><option value="5" >5</option><option value="6" >6</option><option value="7" >7</option><option value="8" >8</option><option value="9" >9</option><option value="10" >10</option><option value="11" >11</option><option value="12" >12</option><option value="13" >13</option><option value="14" >14</option><option value="15" >15</option><option value="16" >16</option><option value="17" >17</option><option value="18" >18</option><option value="19" >19</option><option value="20" >20</option><option value="21" >21</option><option value="22" >22</option><option value="23" >23</option><option value="24" >24</option><option value="25" >25</option><option value="26" >26</option><option value="27" >27</option><option value="28" >28</option><option value="29" >29</option><option value="30" Selected>30</option><option value="31" >31</option>
		</select>
	</td>
	<td>
	<select name="month" size="1" class="formElement" onChange="elementChange(document.stnRequest1,1)" ><option value="1" >Jan</option><option value="2" >Feb</option><option value="3" >Mar</option><option value="4" >Apr</option><option value="5" >May</option><option value="6" >Jun</option><option value="7" >Jul</option><option value="8" >Aug</option><option value="9" >Sep</option><option value="10" >Oct</option><option value="11" Selected>Nov</option><option value="12" >Dec</option>
		</select>
	</td>
	<td>
	<select name="year" size="1" class="formElement" onChange="elementChange(document.stnRequest1,1)"><option value="1998" >1998</option><option value="1999" >1999</option><option value="2000" >2000</option><option value="2001" >2001</option><option value="2002" >2002</option><option value="2003" >2003</option><option value="2004" >2004</option><option value="2005" >2005</option><option value="2006" >2006</option><option value="2007" Selected>2007</option>
	</select>
	</td>
	<td>
	<input type="submit" name="stnSubmit" value="Go" class="formElement">
</td>
</form>

if ($chunk =~ /.*StationID.*?"(\d+)".*?prov.*?"(\w+).*?TableRowHeader">(.*?)<.*abbr title.*?>(\w+).*?/s) {
     my $stationid = $1;
     my $province = $2;
     my $name = $3;
     my $abbrprov = $4;
}

re.match(line, r'^(?P<time>.*?)\s+\[(?P<engine>\d+)\]\s+(?P<error>.*?(KGFXENG|LeadTools).*)'

message = "ERROR: %s %s: %s" % (re.sub(r'.*?([A-Za-z]+Engine[A-Za-z]*?)(Errors)?.*', r'\1', 
                         logfilename), 
                         engine, 
                         match.group('error'))

#Match plain ol' timezones
if ($brpos =~ /^\[(\w+)\](.*)/)
{
	$DateZone = $1;
	$newname = $2;
}
#Match timezones with a day modification, and grab that along with the +/-
elsif ($brpos =~ /^\[(\w+)(\S\d+)\](.*)/)
{
	$DateZone = $1;
	$TempDay2 = ONE_DAY * $2;
	$newname = $3;
}
#Check for a delete flag
elsif ($brpos =~ /^(\d)\[.*/)
{
	$DeleteFilesStatus = $1;
	#If the status is one, we want to capture everything after the timezone as the DeleteName
	if ($DeleteFilesStatus == 1)
	{
		if ($brpos =~ /^(\d)\[(\w+)\](.*)/)
		{
			$DeleteFilesStatus = $1;
			$DateZone = $2;
			$DeleteFilesNames = $3;
			$newname = $3;
		}
		elsif ($brpos =~ /^(\d)\[(\w+)(\S\d+)\](.*)/)
		{
			$DeleteFilesStatus = $1;
			$DateZone = $2;
			$TempDay2 = ONE_DAY * $3;
			$DeleteFilesNames = $4;
			$newname = $4;
		}
	}
	#Otherwise, the DeleteName is in more brackets
	elsif ($DeleteFilesStatus == 2)
	{
                #Grab it all, but without a time modification
		if ($brpos =~ /^(\d)\[(\w+)\]\[(.*\.\w+)\](.*)/)
		{
			$DeleteFilesStatus = $1;
			$DateZone = $2;
			$DeleteFilesNames = $3;
			$newname = $4;
		}
                #Grab it with a time modification
		elsif ($brpos =~ /^(\d)\[(\w+)(\S\d+)\]\[(.*\.\w+)\](.*)/)
		{
			$DeleteFilesStatus = $1;
			$DateZone = $2;
			$TempDay2 = ONE_DAY * $3;
			$DeleteFilesNames = $4;
			$newname = $5;
		}
	}
}

public static string RemoveWhitespace(string str)
{
    try
    {
        return new Regex(@"(\s+|\{.*?\}|\(.*?\)|\/+|\.+)").Replace(str, String.Empty);
    }
    catch (Exception)
    {
        return str;
    }
}

replaceMe = RemoveWhitespace(replaceMe);
Console.WriteLine(replaceMe);
if (replaceMe != String.Empty) 
{
    XmlElement baseElement = Roster.CreateElement(replaceMe);
 
    foreach (XmlElement parseElement in thisElement)
    {
        if (parseElement.HasChildNodes && parseElement.InnerXml.Contains("entity"))
        {
 
            ParseNestedXML(parseElement, baseElement);
        }
        else
        {
            foreach (XmlElement correctElement in parseElement)
            {
                if (correctElement.HasAttribute("name"))
                {
                    string subReplaceMe = correctElement.GetAttribute("name").ToString();
 
                    subReplaceMe = RemoveWhitespace(subReplaceMe);
 
                    XmlElement addElement = Roster.CreateElement(subReplaceMe);
 
                    if (correctElement.HasAttribute("description"))
                    {
                        addElement.InnerText = correctElement.GetAttribute("description").ToString();
                        baseElement.AppendChild(addElement);
                    }
                    //Bye, stats with a value of zero or a hyphen!
                    else if (correctElement.HasAttribute("value") && (Regex.Match(correctElement.GetAttribute("value").ToString(), @"[^0|-]").Success))
                    {
                        addElement.InnerText = correctElement.GetAttribute("value").ToString();
                        baseElement.AppendChild(addElement);
                    }
 
                }
            }
        }
 
        rosterElement.AppendChild(baseElement);
 
        rootElement.AppendChild(rosterElement);
    }
}

def parseitem(d, addto)
  added = addto.add_element(d.attributes["name"].gsub(/\s+/, '')).add_text(d.attributes["description"].gsub(/\./, ''))
  d.elements.each('link') do |ele|
    unless ele.attributes["name"] =~ /(Worker|Helper|Cost|Left)/
      if !ele.attributes["description"].nil?
        added.add_element(ele.attributes["name"].gsub(/\s+/, '')).add_text(ele.attributes["description"])
      else
        #This is necessary for some Gifts of Khaine, apparently
        if added.text == ele.attributes["name"]
          print "Found duplicated #{added.text}!\n\n"
          added.text = ''
        end
        added.add_element(ele.attributes["name"].gsub(/\s+/, '')).add_text("True")
      end
    end
  end
end
 
def parsenested(process, addto)
  #Try to guess if it's a champion, character in the unit, or item
  process.elements.each('entity') do |d|
    if d.attributes["statset"] =~ /Normal/
      #It's a character, crew, or mount.  Figure out which
      if d.attributes["totalcost"] !~ /^0/
        #It's a champion or character
        adder = addto.add_element("champion")
        puts "Found champ\n"
        parse(d, adder)
      else
        #It's crew or the like
        adder = addto.add_element("crew")
        puts "Found crew\n"
        parse(d, adder)
      end
    else
      #It's an item
      puts "Found item\n"
      if addto.elements["item"].nil?
        @adder = addto.add_element("item")
      end
      parseitem(d, @adder)
    end
  end
end

#!/usr/bin/ruby
require "rexml/document"
require "pp"
require "rexml/formatters/default"
include REXML
 
inputxml = File.read('dwarfroster.rst')
@roster = Document.new inputxml
 
@army = Document.new.add_element("army")
 
def parsenested(process, addto)
  #Try to guess if it's a champion, character in the unit, or item
  process.elements.each('entity') do |p|
    #puts p
    if p.elements["link"].has_elements?
      #Recursively run through these to figure out what the hell it is
      if p.elements["link/entity"].attributes["itemsummary"].any?
         adder = addto.add_element("item")
         puts "Found nested\n"
         parsenested(p.elements["link"], adder)
      else
        #This is really just stubbed out, since I haven't seen it
      end
    elsif p.attributes["statset"] =~ /Normal/
      #It's a character, crew, or mount.  Figure out which
      if p.attributes["totalcost"] !~ /^0/
        #It's a champion or character
        adder = addto.add_element("champion")
        puts "Found champ\n"
        parse(p, adder)
      else
        #It's crew or the like
        adder = addto.add_element("crew")
        puts "Found crew\n"
        parse(p, adder)
      end
    else
      #It's an item
      puts "Found item\n"
      if addto.elements["item"].nil?
        @adder = addto.add_element("item")
      end
      added = @adder.add_element(p.attributes["name"].gsub(/\s+/, ''))
      p.elements.each('link') do |ele|
        unless ele.attributes["name"] =~ /(Worker|Helper|Cost|Left)/
          added.add_element(ele.attributes["name"].gsub(/\s+/, '')).add_text(ele.attributes["description"])
        end
      end
    end
  end
end
 
def parse(s, addto)
    #In some cases, the basename differs (i.e. Supreme Sorc vs. High Sorc)
    #Also, it'll pick up whether there's a champion in the unit by the diff
    #of base and count
    %w[basename count base].each do |b|
      if s.attributes[b].any?
        addto.add_element(b).add_text(s.attributes[b])
      end
    end
    stats = addto.add_element("stats")
    s.elements.each('unitstat') do |a|
      #unit.fetch(:stats) { |el| unit[el] = {}}
      #I don't want blank stats
      if a.attributes["value"].any? && (a.attributes["value"] !~ /(0|-)/)
         stats.add_element(a.attributes["name"].gsub(/\s+/, '')).add_text(a.attributes["value"])
      end
    end
    s.elements.each('link') do |link|
      if link.has_elements?
        #Figure out what the hell it is
        parsenested(link, addto)
      else
        #unitatt = unit.add_element("attributes")
        #Rip out the name if it doesn't have "Helper, Worker, Points Left, or Cost"
        unless link.attributes["name"] =~ /(Worker|Helper|Cost|Left)/
          #Get rid of the stuff in braces AB puts in
          if addto.elements["attributes"].nil?
            @unitatt = addto.add_element("attributes")
          end
          @unitatt.add_element(link.attributes["name"].gsub(/\{.*?\}/, '').gsub(/\s+/, '')).add_text('true')
        end
 
      end
    end
end
 
@roster.elements.each('document/roster') do |ele|
  info = @army.add_element("info")
  #Pick out the race, army name, total points, used points, canonical race name
  %w[race size activesize racename].each do |attr|
    info.add_element(attr).add_text(ele.attributes[attr])
  end
  #@army.push(info)
end
 
@roster.elements.each('document/squad') do |ele|
  @unit = @army.add_element("unit")
 
   #Pick out the name of the model and its cost, plus how many models
   %w[name modelcount totalcost].each do |attr|
     @unit.add_element(attr).add_text(ele.attributes[attr])
    end
    ele.elements.each('entity') do |s|
      #Parse it out
      parse(s, @unit)
    end
end
#pp @army
 
prettyprint = REXML::Formatters::Pretty.new
output = String.new
puts prettyprint.write(@army, output)

<?xml version="1.0" encoding="ISO-8859-1"?>
<army>
	<info>
		<race>Dwarf</race>
		<size>1500</size>
		<activesize>1499.</activesize>
		<racename>Dwarfs</racename>
	</info>
	<unit>
		<name>Thane</name>
		<modelcount>1</modelcount>
		<totalcost>134</totalcost>
		<basename>Thane</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>3+</Save>
			<St>4/8</St>
			<To>5</To>
			<UnitSt.>1</UnitSt.>
			<WS>6</WS>
			<Wo>2</Wo>
			<At>3</At>
			<BS>4</BS>
			<In>3</In>
			<ItemPts>75</ItemPts>
		</stats>
		<attributes>
			<General>true</General>
			<HandWeapon>true</HandWeapon>
			<GreatWeapon>true</GreatWeapon>
			<GromrilArmor>true</GromrilArmor>
		</attributes>
		<item>
			<RunicWeapon>
				<MasterRuneofKraggtheGrim>Allows other runes to be placed on a Great Weapon.</MasterRuneofKraggtheGrim>
				<RuneofCleaving>+1 Strength</RuneofCleaving>
			</RunicWeapon>
			<RunicArmor>
				<RuneofStone>+1 Armor Save</RuneofStone>
			</RunicArmor>
		</item>
	</unit>
	<unit>
		<name>Thane</name>
		<modelcount>1</modelcount>
		<totalcost>132</totalcost>
		<basename>Thane</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<In>3</In>
			<ItemPts>75</ItemPts>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>2+/1+</Save>
			<St>4/7</St>
			<To>5</To>
			<UnitSt.>1</UnitSt.>
			<WS>6</WS>
			<Wo>2</Wo>
			<At>3</At>
			<BS>4</BS>
		</stats>
		<attributes>
			<HandWeapon>true</HandWeapon>
			<GromrilArmor>true</GromrilArmor>
			<Shield>true</Shield>
		</attributes>
		<item>
			<RunicWeapon>
				<RuneofCleaving>+1 Strength</RuneofCleaving>
			</RunicWeapon>
			<RunicArmor>
				<RuneofStone>+1 Armor Save</RuneofStone>
			</RunicArmor>
		</item>
	</unit>
	<unit>
		<name>Thane</name>
		<modelcount>1</modelcount>
		<totalcost>95</totalcost>
		<basename>Thane</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<In>3</In>
			<ItemPts>75</ItemPts>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>3+</Save>
			<St>4</St>
			<To>5</To>
			<UnitSt.>1</UnitSt.>
			<WS>6</WS>
			<Wo>2</Wo>
			<At>3</At>
			<BS>4</BS>
		</stats>
		<attributes>
			<HandWeapon>true</HandWeapon>
			<GromrilArmor>true</GromrilArmor>
			<BattleStandardBearer>true</BattleStandardBearer>
		</attributes>
		<item>
			<RunicArmor>
				<RuneofStone>+1 Armor Save</RuneofStone>
			</RunicArmor>
		</item>
	</unit>
	<unit>
		<name>Dwarf Warriors</name>
		<modelcount>20</modelcount>
		<totalcost>205</totalcost>
		<basename>Dwarf Warriors</basename>
		<count>19</count>
		<base>20</base>
		<stats>
			<In>2</In>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>4+/3+</Save>
			<St>3</St>
			<To>4</To>
			<UnitSt.>1</UnitSt.>
			<WS>4</WS>
			<Wo>1</Wo>
			<At>1</At>
			<BS>3</BS>
		</stats>
		<champion>
			<basename>Veteran</basename>
			<count>1</count>
			<base>1</base>
			<stats>
				<In>2</In>
				<Ld>9</Ld>
				<Mv>3</Mv>
				<Save>4+/3+</Save>
				<St>3</St>
				<To>4</To>
				<UnitSt.>1</UnitSt.>
				<WS>4</WS>
				<Wo>1</Wo>
				<At>2</At>
				<BS>3</BS>
			</stats>
			<attributes>
				<HandWeapon>true</HandWeapon>
				<HeavyArmor>true</HeavyArmor>
				<Shield>true</Shield>
			</attributes>
		</champion>
		<attributes>
			<Musician>true</Musician>
			<StandardBearer>true</StandardBearer>
			<HandWeapon>true</HandWeapon>
			<HeavyArmor>true</HeavyArmor>
			<Shield>true</Shield>
		</attributes>
	</unit>
	<unit>
		<name>Quarellers</name>
		<modelcount>10</modelcount>
		<totalcost>110</totalcost>
		<basename>Quarrellers</basename>
		<count>10</count>
		<base>10</base>
		<stats>
			<In>2</In>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>6+</Save>
			<St>3</St>
			<To>4</To>
			<UnitSt.>1</UnitSt.>
			<WS>4</WS>
			<Wo>1</Wo>
			<At>1</At>
			<BS>3</BS>
		</stats>
		<attributes>
			<HandWeapon>true</HandWeapon>
			<Crossbow>true</Crossbow>
			<LightArmor>true</LightArmor>
		</attributes>
	</unit>
	<unit>
		<name>Quarellers</name>
		<modelcount>10</modelcount>
		<totalcost>110</totalcost>
		<basename>Quarrellers</basename>
		<count>10</count>
		<base>10</base>
		<stats>
			<In>2</In>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>6+</Save>
			<St>3</St>
			<To>4</To>
			<UnitSt.>1</UnitSt.>
			<WS>4</WS>
			<Wo>1</Wo>
			<At>1</At>
			<BS>3</BS>
		</stats>
		<attributes>
			<HandWeapon>true</HandWeapon>
			<Crossbow>true</Crossbow>
			<LightArmor>true</LightArmor>
		</attributes>
	</unit>
	<unit>
		<name>Ironbreakers</name>
		<modelcount>14</modelcount>
		<totalcost>237</totalcost>
		<basename>Ironbreakers</basename>
		<count>13</count>
		<base>14</base>
		<stats>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>3+/2+</Save>
			<St>4</St>
			<To>4</To>
			<UnitSt.>1</UnitSt.>
			<WS>5</WS>
			<Wo>1</Wo>
			<At>1</At>
			<BS>3</BS>
			<In>2</In>
		</stats>
		<champion>
			<basename>Ironbeard</basename>
			<count>1</count>
			<base>1</base>
			<stats>
				<In>2</In>
				<Ld>9</Ld>
				<Mv>3</Mv>
				<Save>3+/2+</Save>
				<St>4</St>
				<To>4</To>
				<UnitSt.>1</UnitSt.>
				<WS>5</WS>
				<Wo>1</Wo>
				<At>2</At>
				<BS>3</BS>
			</stats>
			<attributes>
				<HandWeapon>true</HandWeapon>
				<GromrilArmor>true</GromrilArmor>
				<Shield>true</Shield>
			</attributes>
		</champion>
		<attributes>
			<Musician>true</Musician>
			<StandardBearer>true</StandardBearer>
			<HandWeapon>true</HandWeapon>
			<GromrilArmor>true</GromrilArmor>
			<Shield>true</Shield>
		</attributes>
		<item>
			<RunicStandard>
				<RuneofStoicism>The unit counts as double its actual Unit Strength.</RuneofStoicism>
			</RunicStandard>
		</item>
	</unit>
	<unit>
		<name>Hammerers</name>
		<modelcount>18</modelcount>
		<totalcost>246</totalcost>
		<basename>Hammerers</basename>
		<count>17</count>
		<base>18</base>
		<stats>
			<Ld>9</Ld>
			<Mv>3</Mv>
			<Save>5+</Save>
			<St>4/6</St>
			<To>4</To>
			<UnitSt.>1</UnitSt.>
			<WS>5</WS>
			<Wo>1</Wo>
			<At>1</At>
			<BS>3</BS>
			<In>2</In>
		</stats>
		<champion>
			<basename>Gate Keeper</basename>
			<count>1</count>
			<base>1</base>
			<stats>
				<In>2</In>
				<Ld>9</Ld>
				<Mv>3</Mv>
				<Save>5+</Save>
				<St>4/6</St>
				<To>4</To>
				<UnitSt.>1</UnitSt.>
				<WS>5</WS>
				<Wo>1</Wo>
				<At>2</At>
				<BS>3</BS>
			</stats>
			<attributes>
				<HandWeapon>true</HandWeapon>
				<GreatWeapon>true</GreatWeapon>
				<HeavyArmor>true</HeavyArmor>
			</attributes>
		</champion>
		<attributes>
			<Musician>true</Musician>
			<StandardBearer>true</StandardBearer>
			<HandWeapon>true</HandWeapon>
			<GreatWeapon>true</GreatWeapon>
			<HeavyArmor>true</HeavyArmor>
			<Stubborn>true</Stubborn>
		</attributes>
	</unit>
	<unit>
		<name>Artillery Battery</name>
		<modelcount>4</modelcount>
		<totalcost>45</totalcost>
		<basename>Bolt Thrower</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<To>7</To>
			<UnitSt.>3</UnitSt.>
			<Wo>3</Wo>
		</stats>
		<crew>
			<basename>Crew</basename>
			<count>3</count>
			<base>3</base>
			<stats>
				<In>2</In>
				<Ld>9</Ld>
				<Mv>3</Mv>
				<Save>6+</Save>
				<St>3</St>
				<To>4</To>
				<WS>4</WS>
				<Wo>1</Wo>
				<At>1</At>
				<BS>3</BS>
			</stats>
			<attributes>
				<HandWeapon>true</HandWeapon>
				<LightArmor>true</LightArmor>
			</attributes>
		</crew>
		<attributes>
			<BoltThrower>true</BoltThrower>
		</attributes>
	</unit>
	<unit>
		<name>Artillery Battery</name>
		<modelcount>4</modelcount>
		<totalcost>45</totalcost>
		<basename>Bolt Thrower</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<To>7</To>
			<UnitSt.>3</UnitSt.>
			<Wo>3</Wo>
		</stats>
		<crew>
			<basename>Crew</basename>
			<count>3</count>
			<base>3</base>
			<stats>
				<In>2</In>
				<Ld>9</Ld>
				<Mv>3</Mv>
				<Save>6+</Save>
				<St>3</St>
				<To>4</To>
				<WS>4</WS>
				<Wo>1</Wo>
				<At>1</At>
				<BS>3</BS>
			</stats>
			<attributes>
				<HandWeapon>true</HandWeapon>
				<LightArmor>true</LightArmor>
			</attributes>
		</crew>
		<attributes>
			<BoltThrower>true</BoltThrower>
		</attributes>
	</unit>
	<unit>
		<name>Airborne Assault</name>
		<modelcount>1</modelcount>
		<totalcost>140</totalcost>
		<basename>Gyrocopter</basename>
		<count>1</count>
		<base>1</base>
		<stats>
			<In>2</In>
			<Ld>9</Ld>
			<Save>4+</Save>
			<St>4</St>
			<To>5</To>
			<UnitSt.>3</UnitSt.>
			<WS>4</WS>
			<Wo>3</Wo>
			<At>2</At>
		</stats>
		<attributes>
			<Flyer>true</Flyer>
		</attributes>
	</unit>
</army>

<item>
      <SacrificialDagger/>
      <PearlofinfiniteBleakeness/>
      <BlackDragonEgg/>
</item>
<item>
        <RuneofKhaine/>
        <TouchofDeath>
          <KillingBlow/>
        </TouchofDeath>
</item>

<link id="dwWarCrew" count="1" actual="1" script="0" sequence="106" pseudo="no" totalcost="0" \ 
name="Crew" category="Equip" visible="no" sourceid="dwBoltThrw" sourceindex="1"></link>

<ruleset context="dwSubtype" ruleset="dwDwarves" contextname="Army Subtype" rulesetname="Dwarf Army"/>

<link id="HeavyArmor" count="1" actual="1" script="0" sequence="26" pseudo="no" totalcost="0" name="Heavy Armor" category="Equip" \
 abbrev="Hv" description="5+ Armor Save" equipment="yes" footnote="yes" sourceid="dwWarrVet" sourceindex="5"></link>

<link id="ItemCost" count="1" actual="1" script="0" sequence="28" pseudo="no" totalcost="0" name="Item Cost Worker" category="Equip"\
 visible="no" sourcetype="3" sourceid="Globals" sourceindex="1"></link>

<document>
  <squad>
  <!-- unit name and cost is here -->
    <entity>
    <!-- unit stats are here, along with composition and whatnot -->
      <link>
      <!-- sometimes there's nothing of note in the link tags -->
         <entity>
         <!-- this might be a magic item, warmachine crew, magic banner, champion, and probably other stuff, but is not easily  \
         identified, and there may be more than one -->
            <link>
            <!-- might be info for whatever is in entity, might be a helper which I don't want -->
            </link>
            <unitstat>
            <!-- if it's crew, champion, whatever, stats would be here, but this node may not exist -->
            </unitstat>
        </entity>
      </link>
   </entity>
 </squad>
</document>